This is an automated email from the ASF dual-hosted git repository.
peacewong pushed a commit to branch dev-1.3.1
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git
The following commit(s) were added to refs/heads/dev-1.3.1 by this push:
new a4edf1b06 [Linkis-datasource] Adjust the architecture of metadata
service and add support of HDFS type in datasource (#3613)
a4edf1b06 is described below
commit a4edf1b0616beb70fc0590223c1e335f8004ec22
Author: David hua <[email protected]>
AuthorDate: Thu Oct 20 11:45:58 2022 +0800
[Linkis-datasource] Adjust the architecture of metadata service and add
support of HDFS type in datasource (#3613)
---
.../datasourcemanager/common/DataSources.java | 56 +++++++
...aService.java => AbstractCacheMetaService.java} | 177 ++++++---------------
.../common/service/AbstractDbMetaService.java | 152 ++++++++++++++++++
...dataService.java => AbstractFsMetaService.java} | 23 +--
.../query/common/service/BaseMetadataService.java | 14 ++
...MetadataService.java => MetadataFsService.java} | 16 +-
.../server/src/main/assembly/distribution.xml | 11 ++
.../server/loader/MetaClassLoaderManager.java | 85 ++++++++--
.../query/server/restful/MetadataQueryRestful.java | 26 +++
.../query/server/service/MetadataQueryService.java | 13 ++
.../service/impl/MetadataQueryServiceImpl.java | 44 ++++-
.../metadata/query/server/utils/MetadataUtils.java | 40 ++++-
.../metadata/query/service/EsMetaService.java | 4 +-
.../linkis-metadata-query/service/hdfs/pom.xml | 110 +++++++++++++
.../hdfs/src/main/assembly/distribution.xml | 57 +++++++
.../metadata/query/service/HdfsConnection.java | 108 +++++++++++++
.../metadata/query/service/HdfsMetaService.java | 121 ++++++++++++++
.../metadata/query/service/HdfsParamsMapper.java} | 20 +--
.../query/service/conf/ConfigurationUtils.java | 141 ++++++++++++++++
.../metadata/query/service/HiveMetaService.java | 4 +-
.../metadata/query/service/KafkaMetaService.java | 4 +-
.../metadata/query/service/Db2MetaService.java | 4 +-
.../metadata/query/service/DmMetaService.java | 4 +-
.../query/service/GreenplumMetaService.java | 4 +-
.../query/service/KingbaseMetaService.java | 4 +-
.../metadata/query/service/MysqlMetaService.java | 4 +-
.../metadata/query/service/OracleMetaService.java | 4 +-
.../query/service/PostgresqlMetaService.java | 4 +-
.../query/service/SqlserverMetaService.java | 4 +-
.../linkis-datasource/pom.xml | 1 +
30 files changed, 1058 insertions(+), 201 deletions(-)
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/common/src/main/java/org/apache/linkis/datasourcemanager/common/DataSources.java
b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/common/src/main/java/org/apache/linkis/datasourcemanager/common/DataSources.java
new file mode 100644
index 000000000..d6fb45a01
--- /dev/null
+++
b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/common/src/main/java/org/apache/linkis/datasourcemanager/common/DataSources.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.datasourcemanager.common;
+
+import org.apache.linkis.common.conf.CommonVars;
+import org.apache.linkis.datasourcemanager.common.domain.DataSource;
+import org.apache.linkis.datasourcemanager.common.domain.DataSourceType;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class DataSources {
+
+ /** Default HDFS name */
+ private static final CommonVars<String> DEFAULT_HDFS_NAME =
+ CommonVars.apply("wds.linkis.server.dsm.default.hdfs.name",
".LOCAL_HDFS");
+
+ private static final Map<String, DataSource> DEFAULT_DATASOURCES = new
HashMap<>();
+
+ static {
+ DataSourceType hdfsType = new DataSourceType();
+ hdfsType.setName("hdfs");
+ DataSource hdfs = new DataSource();
+ hdfs.setDataSourceType(hdfsType);
+ hdfs.setDataSourceName(DEFAULT_HDFS_NAME.getValue());
+ DEFAULT_DATASOURCES.put(hdfs.getDataSourceName(), hdfs);
+ DEFAULT_DATASOURCES
+ .values()
+ .forEach(dataSource ->
dataSource.setCreateUser(System.getProperty("user.name")));
+ }
+
+ /**
+ * Find the default data source by name
+ *
+ * @param dataSourceName data source name
+ * @return data source
+ */
+ public static DataSource getDefault(String dataSourceName) {
+ return DEFAULT_DATASOURCES.get(dataSourceName);
+ }
+}
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/AbstractMetaService.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/AbstractCacheMetaService.java
similarity index 62%
rename from
linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/AbstractMetaService.java
rename to
linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/AbstractCacheMetaService.java
index bde919771..5af1a3fbe 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/AbstractMetaService.java
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/AbstractCacheMetaService.java
@@ -17,13 +17,10 @@
package org.apache.linkis.metadata.query.common.service;
-import org.apache.linkis.common.exception.WarnException;
import org.apache.linkis.datasourcemanager.common.util.json.Json;
import org.apache.linkis.metadata.query.common.cache.CacheConfiguration;
import org.apache.linkis.metadata.query.common.cache.CacheManager;
import org.apache.linkis.metadata.query.common.cache.ConnCacheManager;
-import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo;
-import org.apache.linkis.metadata.query.common.domain.MetaPartitionInfo;
import org.apache.linkis.metadata.query.common.exception.MetaRuntimeException;
import org.apache.commons.lang3.StringUtils;
@@ -34,7 +31,7 @@ import java.io.Closeable;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
-import java.util.List;
+import java.util.Collections;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.Callable;
@@ -45,8 +42,15 @@ import com.google.common.cache.Cache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public abstract class AbstractMetaService<C extends Closeable> implements
MetadataService {
- private static final Logger LOG =
LoggerFactory.getLogger(AbstractMetaService.class);
+/**
+ * Meta service use cache manager
+ *
+ * @param <C>
+ */
+public abstract class AbstractCacheMetaService<C extends Closeable> implements
BaseMetadataService {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(AbstractCacheMetaService.class);
+
private static final String CONN_CACHE_REQ = "_STORED";
private CacheManager connCacheManager;
@@ -62,140 +66,40 @@ public abstract class AbstractMetaService<C extends
Closeable> implements Metada
/**
* If want to use cache component, you should invoke this in constructor
method
*
- * @param cacheManager
+ * @param cacheManager cache manage
*/
protected void initCache(CacheManager cacheManager) {
- String prefix = this.getClass().getSimpleName();
- reqCache =
- cacheManager.buildCache(
- prefix + CONN_CACHE_REQ,
- notification -> {
- assert notification.getValue() != null;
- close(notification.getValue().getConnection());
- });
- // Clean up the req cache
- reqCache.cleanUp();
- }
-
- @Override
- public abstract MetadataConnection<C> getConnection(String operator,
Map<String, Object> params)
- throws Exception;
-
- @Override
- public List<String> getDatabases(String operator, Map<String, Object>
params) {
- return this.getConnAndRun(operator, params, this::queryDatabases);
- }
-
- @Override
- public List<String> getTables(String operator, Map<String, Object> params,
String database) {
- return this.getConnAndRun(operator, params, conn -> this.queryTables(conn,
database));
- }
-
- @Override
- public Map<String, String> getTableProps(
- String operator, Map<String, Object> params, String database, String
table) {
- return this.getConnAndRun(
- operator, params, conn -> this.queryTableProps(conn, database, table));
- }
-
- @Override
- public MetaPartitionInfo getPartitions(
- String operator,
- Map<String, Object> params,
- String database,
- String table,
- boolean traverse) {
- return this.getConnAndRun(
- operator, params, conn -> this.queryPartitions(conn, database, table,
traverse));
- }
-
- @Override
- public List<MetaColumnInfo> getColumns(
- String operator, Map<String, Object> params, String database, String
table) {
- return this.getConnAndRun(operator, params, conn ->
this.queryColumns(conn, database, table));
- }
-
- @Override
- public Map<String, String> getPartitionProps(
- String operator,
- Map<String, Object> params,
- String database,
- String table,
- String partition) {
- return this.getConnAndRun(
- operator, params, conn -> this.queryPartitionProps(conn, database,
table, partition));
- }
-
- /**
- * Get database list by connection
- *
- * @param connection metadata connection
- * @return
- */
- public List<String> queryDatabases(C connection) {
- throw new WarnException(-1, "This method is no supported");
- }
-
- /**
- * Get table list by connection and database
- *
- * @param connection metadata connection
- * @param database database
- * @return
- */
- public List<String> queryTables(C connection, String database) {
- throw new WarnException(-1, "This method is no supported");
- }
-
- /**
- * Get partitions by connection, database and table
- *
- * @param connection metadata connection
- * @param database database
- * @param table table
- * @return
- */
- public MetaPartitionInfo queryPartitions(
- C connection, String database, String table, boolean traverse) {
- throw new WarnException(-1, "This method is no supported");
+ if (useCache()) {
+ String prefix = this.getClass().getSimpleName();
+ reqCache =
+ cacheManager.buildCache(
+ prefix + CONN_CACHE_REQ,
+ notification -> {
+ assert notification.getValue() != null;
+ close(notification.getValue().getConnection());
+ });
+ // Clean up the req cache
+ reqCache.cleanUp();
+ }
}
/**
- * Get columns by connection, database and table
+ * If use the cache
*
- * @param connection metadata connection
- * @param database database
- * @param table table
- * @return
+ * @return boolean
*/
- public List<MetaColumnInfo> queryColumns(C connection, String database,
String table) {
- throw new WarnException(-1, "This method is no supported");
+ protected boolean useCache() {
+ return true;
}
- /**
- * Get the properties of partition
- *
- * @param connection
- * @param database
- * @param table
- * @param partition
- * @return
- */
- public Map<String, String> queryPartitionProps(
- C connection, String database, String table, String partition) {
- throw new WarnException(-1, "This method is no supported");
- }
+ @Override
+ public abstract MetadataConnection<C> getConnection(String operator,
Map<String, Object> params)
+ throws Exception;
- /**
- * Get table properties
- *
- * @param connection metadata connection
- * @param database database
- * @param table table
- * @return
- */
- public Map<String, String> queryTableProps(C connection, String database,
String table) {
- throw new WarnException(-1, "This method is no supported");
+ public Map<String, String> getConnectionInfo(
+ String operator, Map<String, Object> params, Map<String, String>
queryParams) {
+ return this.getConnAndRun(
+ operator, params, connection -> this.queryConnectionInfo(connection,
queryParams));
}
public void close(C connection) {
@@ -207,12 +111,25 @@ public abstract class AbstractMetaService<C extends
Closeable> implements Metada
}
}
+ /**
+ * Get connection information
+ *
+ * @param connection connection
+ * @param queryParams query params
+ * @return map
+ */
+ public Map<String, String> queryConnectionInfo(C connection, Map<String,
String> queryParams) {
+ return Collections.emptyMap();
+ }
+
protected <R> R getConnAndRun(
String operator, Map<String, Object> params, Function<C, R> action) {
String cacheKey = "";
MetadataConnection<C> connection = null;
try {
cacheKey = md5String(Json.toJson(params, null), "", 2);
+ // Dive the cache by operator/creator
+ cacheKey = operator + "_" + md5String(Json.toJson(params, null), "", 2);
if (null != reqCache) {
ConnectionCache<C> connectionCache =
getConnectionInCache(reqCache, cacheKey, () ->
getConnection(operator, params));
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/AbstractDbMetaService.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/AbstractDbMetaService.java
new file mode 100644
index 000000000..3e113909d
--- /dev/null
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/AbstractDbMetaService.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.metadata.query.common.service;
+
+import org.apache.linkis.common.exception.WarnException;
+import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo;
+import org.apache.linkis.metadata.query.common.domain.MetaPartitionInfo;
+
+import java.io.Closeable;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Database meta service
+ *
+ * @param <C>
+ */
+public abstract class AbstractDbMetaService<C extends Closeable> extends
AbstractCacheMetaService<C>
+ implements MetadataDbService {
+
+ @Override
+ public List<String> getDatabases(String operator, Map<String, Object>
params) {
+ return this.getConnAndRun(operator, params, this::queryDatabases);
+ }
+
+ @Override
+ public List<String> getTables(String operator, Map<String, Object> params,
String database) {
+ return this.getConnAndRun(operator, params, conn -> this.queryTables(conn,
database));
+ }
+
+ @Override
+ public Map<String, String> getTableProps(
+ String operator, Map<String, Object> params, String database, String
table) {
+ return this.getConnAndRun(
+ operator, params, conn -> this.queryTableProps(conn, database, table));
+ }
+
+ @Override
+ public MetaPartitionInfo getPartitions(
+ String operator,
+ Map<String, Object> params,
+ String database,
+ String table,
+ boolean traverse) {
+ return this.getConnAndRun(
+ operator, params, conn -> this.queryPartitions(conn, database, table,
traverse));
+ }
+
+ @Override
+ public List<MetaColumnInfo> getColumns(
+ String operator, Map<String, Object> params, String database, String
table) {
+ return this.getConnAndRun(operator, params, conn ->
this.queryColumns(conn, database, table));
+ }
+
+ @Override
+ public Map<String, String> getPartitionProps(
+ String operator,
+ Map<String, Object> params,
+ String database,
+ String table,
+ String partition) {
+ return this.getConnAndRun(
+ operator, params, conn -> this.queryPartitionProps(conn, database,
table, partition));
+ }
+
+ /**
+ * Get database list by connection
+ *
+ * @param connection metadata connection
+ * @return
+ */
+ public List<String> queryDatabases(C connection) {
+ throw new WarnException(-1, "This method is no supported");
+ }
+
+ /**
+ * Get table list by connection and database
+ *
+ * @param connection metadata connection
+ * @param database database
+ * @return
+ */
+ public List<String> queryTables(C connection, String database) {
+ throw new WarnException(-1, "This method is no supported");
+ }
+
+ /**
+ * Get partitions by connection, database and table
+ *
+ * @param connection metadata connection
+ * @param database database
+ * @param table table
+ * @return
+ */
+ public MetaPartitionInfo queryPartitions(
+ C connection, String database, String table, boolean traverse) {
+ throw new WarnException(-1, "This method is no supported");
+ }
+
+ /**
+ * Get columns by connection, database and table
+ *
+ * @param connection metadata connection
+ * @param database database
+ * @param table table
+ * @return
+ */
+ public List<MetaColumnInfo> queryColumns(C connection, String database,
String table) {
+ throw new WarnException(-1, "This method is no supported");
+ }
+
+ /**
+ * Get the properties of partition
+ *
+ * @param connection
+ * @param database
+ * @param table
+ * @param partition
+ * @return
+ */
+ public Map<String, String> queryPartitionProps(
+ C connection, String database, String table, String partition) {
+ throw new WarnException(-1, "This method is no supported");
+ }
+
+ /**
+ * Get table properties
+ *
+ * @param connection metadata connection
+ * @param database database
+ * @param table table
+ * @return
+ */
+ public Map<String, String> queryTableProps(C connection, String database,
String table) {
+ throw new WarnException(-1, "This method is no supported");
+ }
+}
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/BaseMetadataService.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/AbstractFsMetaService.java
similarity index 69%
copy from
linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/BaseMetadataService.java
copy to
linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/AbstractFsMetaService.java
index e2cf804b3..610896bb0 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/BaseMetadataService.java
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/AbstractFsMetaService.java
@@ -20,14 +20,19 @@ package org.apache.linkis.metadata.query.common.service;
import java.io.Closeable;
import java.util.Map;
-public interface BaseMetadataService {
+/**
+ * Filesystem meta service
+ *
+ * @param <C>
+ */
+public abstract class AbstractFsMetaService<C extends Closeable> extends
AbstractCacheMetaService<C>
+ implements MetadataFsService {
+ @Override
+ public String getSchema(String operator, Map<String, Object> params) {
+ return this.getConnAndRun(operator, params, this::getSchema);
+ }
- /**
- * Get connection
- *
- * @param params connect params
- * @return
- */
- MetadataConnection<? extends Closeable> getConnection(String operator,
Map<String, Object> params)
- throws Exception;
+ public String getSchema(C connection) {
+ return "";
+ }
}
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/BaseMetadataService.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/BaseMetadataService.java
index e2cf804b3..e090d6cac 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/BaseMetadataService.java
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/BaseMetadataService.java
@@ -18,6 +18,7 @@
package org.apache.linkis.metadata.query.common.service;
import java.io.Closeable;
+import java.util.Collections;
import java.util.Map;
public interface BaseMetadataService {
@@ -30,4 +31,17 @@ public interface BaseMetadataService {
*/
MetadataConnection<? extends Closeable> getConnection(String operator,
Map<String, Object> params)
throws Exception;
+
+ /**
+ * Get connection information (default empty)
+ *
+ * @param operator operator
+ * @param params connect params
+ * @param queryParams query params
+ * @return information
+ */
+ default Map<String, String> getConnectionInfo(
+ String operator, Map<String, Object> params, Map<String, String>
queryParams) {
+ return Collections.emptyMap();
+ }
}
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/MetadataService.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/MetadataFsService.java
similarity index 74%
rename from
linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/MetadataService.java
rename to
linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/MetadataFsService.java
index 502ff0793..1c3bcb624 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/MetadataService.java
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/MetadataFsService.java
@@ -14,7 +14,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.linkis.metadata.query.common.service;
-public interface MetadataService extends MetadataDbService {}
+import java.util.Map;
+
+/** Metadata Fs service */
+public interface MetadataFsService extends BaseMetadataService {
+
+ /**
+ * Get the File System schema
+ *
+ * @return schema name
+ */
+ default String getSchema(String operator, Map<String, Object> params) {
+ return "file";
+ }
+}
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/assembly/distribution.xml
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/assembly/distribution.xml
index 0387aded7..7c94aa858 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/assembly/distribution.xml
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/assembly/distribution.xml
@@ -95,6 +95,17 @@
<exclude>*-javadoc.jar</exclude>
</excludes>
</fileSet>
+ <fileSet>
+ <directory>${basedir}/../service/hdfs/target/out/lib</directory>
+ <fileMode>0755</fileMode>
+ <outputDirectory>lib/service/hdfs</outputDirectory>
+ <includes>
+ <include>*.jar</include>
+ </includes>
+ <excludes>
+ <exclude>*-javadoc.jar</exclude>
+ </excludes>
+ </fileSet>
</fileSets>
</assembly>
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/loader/MetaClassLoaderManager.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/loader/MetaClassLoaderManager.java
index 4bff6cc2d..097769980 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/loader/MetaClassLoaderManager.java
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/loader/MetaClassLoaderManager.java
@@ -22,8 +22,8 @@ import org.apache.linkis.common.conf.Configuration;
import org.apache.linkis.common.exception.ErrorException;
import org.apache.linkis.metadata.query.common.cache.CacheConfiguration;
import org.apache.linkis.metadata.query.common.exception.MetaRuntimeException;
-import org.apache.linkis.metadata.query.common.service.AbstractMetaService;
-import org.apache.linkis.metadata.query.common.service.MetadataService;
+import
org.apache.linkis.metadata.query.common.service.AbstractCacheMetaService;
+import org.apache.linkis.metadata.query.common.service.BaseMetadataService;
import org.apache.linkis.metadata.query.server.utils.MetadataUtils;
import org.apache.commons.lang3.StringUtils;
@@ -127,17 +127,17 @@ public class MetaClassLoaderManager {
String prefix = dsType.substring(0, 1).toUpperCase() +
dsType.substring(1);
expectClassName = String.format(META_CLASS_NAME, prefix);
}
- Class<? extends MetadataService> metaServiceClass =
+ Class<? extends BaseMetadataService> metaServiceClass =
searchForLoadMetaServiceClass(metaClassLoader,
expectClassName, true);
if (Objects.isNull(metaServiceClass)) {
throw new MetaRuntimeException(
"Fail to init and load meta service class for type: [" +
dsType + "]", null);
}
- MetadataService metadataService =
+ BaseMetadataService metadataService =
MetadataUtils.loadMetaService(metaServiceClass,
metaClassLoader);
- if (metadataService instanceof AbstractMetaService) {
+ if (metadataService instanceof AbstractCacheMetaService) {
LOG.info("Invoke the init() method in meta service for type:
[" + dsType + "]");
- ((AbstractMetaService<?>) metadataService).init();
+ ((AbstractCacheMetaService<?>) metadataService).init();
}
return new MetaServiceInstance(metadataService,
metaClassLoader);
});
@@ -148,12 +148,65 @@ public class MetaClassLoaderManager {
ClassLoader currentClassLoader =
Thread.currentThread().getContextClassLoader();
try {
Thread.currentThread().setContextClassLoader(finalServiceInstance.metaClassLoader);
- Method method =
+ List<Method> methodsMatched =
Arrays.stream(childMethods)
- .filter(eachMethod -> eachMethod.getName().equals(m))
- .collect(Collectors.toList())
- .get(0);
- return method.invoke(finalServiceInstance.serviceInstance, args);
+ .filter(
+ eachMethod -> {
+ if (eachMethod.getName().equals(m)) {
+ Class<?>[] parameterType =
eachMethod.getParameterTypes();
+ if (parameterType.length == args.length) {
+ for (int i = 0; i < parameterType.length; i++) {
+ if (Objects.nonNull(args[i])) {
+ boolean matches =
+
parameterType[i].isAssignableFrom(args[i].getClass())
+ || ((args[i].getClass().isPrimitive()
+ ||
parameterType[i].isPrimitive())
+ &&
MetadataUtils.getPrimitive(args[i].getClass())
+ ==
MetadataUtils.getPrimitive(parameterType[i]));
+ if (!matches) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+ }
+ return false;
+ })
+ .collect(Collectors.toList());
+ if (methodsMatched.isEmpty()) {
+ String type = null;
+ if (Objects.nonNull(args)) {
+ type =
+ Arrays.stream(args)
+ .map(arg -> Objects.nonNull(arg) ?
arg.getClass().toString() : "null")
+ .collect(Collectors.joining(","));
+ }
+ String message =
+ "Unknown method: [ name: "
+ + m
+ + ", type: ["
+ + type
+ + "]] for meta service instance: ["
+ + finalServiceInstance.getServiceInstance().toString()
+ + "]";
+ LOG.warn(message);
+ throw new MetaRuntimeException(message, null);
+ } else if (methodsMatched.size() > 1) {
+ LOG.warn(
+ "Find multiple matched methods with name: ["
+ + m
+ + "] such as: \n"
+ + methodsMatched.stream()
+ .map(
+ method ->
+ method.getName() + ":" +
Arrays.toString(method.getParameterTypes()))
+ .collect(Collectors.joining("\n"))
+ + "\n in meta service instance: ["
+ + finalServiceInstance.getServiceInstance().toString()
+ + "], will choose the first one");
+ }
+ return
methodsMatched.get(0).invoke(finalServiceInstance.serviceInstance, args);
} catch (Exception e) {
Throwable t = e;
// UnWrap the Invocation target exception
@@ -174,12 +227,12 @@ public class MetaClassLoaderManager {
};
}
- private Class<? extends MetadataService> searchForLoadMetaServiceClass(
+ private Class<? extends BaseMetadataService> searchForLoadMetaServiceClass(
ClassLoader classLoader, String expectClassName, boolean initialize) {
ClassLoader currentClassLoader =
Thread.currentThread().getContextClassLoader();
Thread.currentThread().setContextClassLoader(classLoader);
try {
- Class<? extends MetadataService> metaClass = null;
+ Class<? extends BaseMetadataService> metaClass = null;
if (StringUtils.isNotBlank(expectClassName)) {
metaClass =
MetadataUtils.loadMetaServiceClass(
@@ -228,7 +281,7 @@ public class MetaClassLoaderManager {
/** ServiceInstance Holder */
public static class MetaServiceInstance {
- private MetadataService serviceInstance;
+ private BaseMetadataService serviceInstance;
private Method[] methods;
@@ -236,14 +289,14 @@ public class MetaClassLoaderManager {
private long initTimeStamp = 0L;
- public MetaServiceInstance(MetadataService serviceInstance, ClassLoader
metaClassLoader) {
+ public MetaServiceInstance(BaseMetadataService serviceInstance,
ClassLoader metaClassLoader) {
this.serviceInstance = serviceInstance;
this.metaClassLoader = metaClassLoader;
this.methods = serviceInstance.getClass().getMethods();
this.initTimeStamp = System.currentTimeMillis();
}
- public MetadataService getServiceInstance() {
+ public BaseMetadataService getServiceInstance() {
return serviceInstance;
}
}
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/restful/MetadataQueryRestful.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/restful/MetadataQueryRestful.java
index fc653181e..0699bf346 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/restful/MetadataQueryRestful.java
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/restful/MetadataQueryRestful.java
@@ -54,6 +54,32 @@ public class MetadataQueryRestful {
@Autowired private MetadataQueryService metadataQueryService;
+ @RequestMapping(value = "/getConnectionInfo", method = RequestMethod.GET)
+ public Message getConnectionInfo(
+ @RequestParam("dataSourceName") String dataSourceName,
+ @RequestParam("system") String system,
+ HttpServletRequest request){
+ try {
+ if (StringUtils.isBlank(system)){
+ return Message.error("'system' is missing[缺少系统名]");
+ }
+ Map<String, String> queryParams =
request.getParameterMap().entrySet().stream()
+ .collect(Collectors.toMap(Map.Entry::getKey,
+ entry -> StringUtils.join(entry.getValue(), ",")));
+ Map<String, String> info =
metadataQueryService.getConnectionInfoByDsName(dataSourceName, queryParams,
+ system, SecurityFilter.getLoginUsername(request));
+ return Message.ok().data("info", info);
+ } catch (Exception e){
+ return errorToResponseMessage(
+ "Fail to get connection info [获得连接信息失败], name: ["
+ + dataSourceName
+ + "], system:["
+ + system
+ + "]",
+ e);
+ }
+ }
+
@ApiOperation(value = "getDatabases", notes = "get databases", response =
Message.class)
@ApiImplicitParams({
@ApiImplicitParam(name = "dataSourceName", required = true, dataType =
"String", value = "data source name"),
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/service/MetadataQueryService.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/service/MetadataQueryService.java
index cba871ef5..eedd1a0a3 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/service/MetadataQueryService.java
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/service/MetadataQueryService.java
@@ -114,6 +114,19 @@ public interface MetadataQueryService {
String dataSourceId, String database, String table, String system,
String userName)
throws ErrorException;
+ /**
+ * Get connection information
+ *
+ * @param dataSourceName data source name
+ * @param queryParams query params
+ * @param system system
+ * @param userName user
+ * @return
+ */
+ Map<String, String> getConnectionInfoByDsName(
+ String dataSourceName, Map<String, String> queryParams, String system,
String userName)
+ throws ErrorException;
+
/**
* @param dataSourceName data source name
* @param system system
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/service/impl/MetadataQueryServiceImpl.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/service/impl/MetadataQueryServiceImpl.java
index a4e0a81db..e6ed0838f 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/service/impl/MetadataQueryServiceImpl.java
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/service/impl/MetadataQueryServiceImpl.java
@@ -18,7 +18,9 @@
package org.apache.linkis.metadata.query.server.service.impl;
import org.apache.linkis.common.exception.ErrorException;
+import org.apache.linkis.datasourcemanager.common.DataSources;
import org.apache.linkis.datasourcemanager.common.auth.AuthContext;
+import org.apache.linkis.datasourcemanager.common.domain.DataSource;
import org.apache.linkis.datasourcemanager.common.protocol.DsInfoQueryRequest;
import org.apache.linkis.datasourcemanager.common.protocol.DsInfoResponse;
import org.apache.linkis.metadata.query.common.MdmConfiguration;
@@ -196,6 +198,21 @@ public class MetadataQueryServiceImpl implements
MetadataQueryService {
return new ArrayList<>();
}
+ @Override
+ public Map<String, String> getConnectionInfoByDsName(
+ String dataSourceName, Map<String, String> queryParams, String system,
String userName)
+ throws ErrorException {
+ DsInfoResponse dsInfoResponse = queryDataSourceInfoByName(dataSourceName,
system, userName);
+ if (StringUtils.isNotBlank(dsInfoResponse.dsType())) {
+ return invokeMetaMethod(
+ dsInfoResponse.dsType(),
+ "getConnectionInfo",
+ new Object[] {dsInfoResponse.creator(), dsInfoResponse.params(),
queryParams},
+ Map.class);
+ }
+ return new HashMap<>();
+ }
+
@Override
public List<String> getTablesByDsName(
String dataSourceName, String database, String system, String userName)
@@ -333,8 +350,14 @@ public class MetadataQueryServiceImpl implements
MetadataQueryService {
public DsInfoResponse queryDataSourceInfoByName(
String dataSourceName, String system, String userName) throws
ErrorException {
Object rpcResult = null;
+ boolean useDefault = false;
try {
- rpcResult = dataSourceRpcSender.ask(new DsInfoQueryRequest(null,
dataSourceName, system));
+ rpcResult = reqGetDefaultDataSource(dataSourceName);
+ if (Objects.isNull(rpcResult)) {
+ rpcResult = dataSourceRpcSender.ask(new DsInfoQueryRequest(null,
dataSourceName, system));
+ } else {
+ useDefault = true;
+ }
} catch (Exception e) {
throw new ErrorException(-1, "Remote Service Error[远端服务出错, 联系运维处理]");
}
@@ -349,7 +372,7 @@ public class MetadataQueryServiceImpl implements
MetadataQueryService {
&& userName.equals(response.creator())));
if (!hasPermission) {
throw new ErrorException(-1, "Don't have query permission for data
source [没有数据源的查询权限]");
- } else if (response.params().isEmpty()) {
+ } else if (!useDefault && response.params().isEmpty()) {
throw new ErrorException(-1, "Have you published the data source?
[数据源未发布或者参数为空]");
}
return response;
@@ -358,6 +381,23 @@ public class MetadataQueryServiceImpl implements
MetadataQueryService {
}
}
+ /**
+ * Request to get default data source
+ *
+ * @param dataSourceName data source name
+ * @return response
+ */
+ private DsInfoResponse reqGetDefaultDataSource(String dataSourceName) {
+ DataSource dataSource = DataSources.getDefault(dataSourceName);
+ return (Objects.nonNull(dataSource))
+ ? new DsInfoResponse(
+ true,
+ dataSource.getDataSourceType().getName(),
+ dataSource.getConnectParams(),
+ dataSource.getCreateUser())
+ : null;
+ }
+
/**
* Invoke method in meta service
*
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/utils/MetadataUtils.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/utils/MetadataUtils.java
index 0e3b69a8e..fc748ea84 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/utils/MetadataUtils.java
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/utils/MetadataUtils.java
@@ -19,7 +19,7 @@ package org.apache.linkis.metadata.query.server.utils;
import org.apache.linkis.common.conf.CommonVars;
import org.apache.linkis.metadata.query.common.exception.MetaRuntimeException;
-import org.apache.linkis.metadata.query.common.service.MetadataService;
+import org.apache.linkis.metadata.query.common.service.BaseMetadataService;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
@@ -54,8 +54,31 @@ public class MetadataUtils {
public static final Pattern nameRegexPattern = Pattern.compile(NAME_REGEX);
- public static MetadataService loadMetaService(
- Class<? extends MetadataService> metaServiceClass, ClassLoader
metaServiceClassLoader) {
+ /**
+ * Get the primitive class
+ *
+ * @param clazz class
+ * @return return
+ */
+ public static Class<?> getPrimitive(Class<?> clazz) {
+ try {
+ Class<?> primitive = null;
+ if (clazz.isPrimitive()) {
+ primitive = clazz;
+ } else {
+ Class<?> innerType = ((Class<?>) clazz.getField("TYPE").get(null));
+ if (innerType.isPrimitive()) {
+ primitive = innerType;
+ }
+ }
+ return primitive;
+ } catch (NoSuchFieldException | IllegalAccessException e) {
+ return null;
+ }
+ }
+
+ public static BaseMetadataService loadMetaService(
+ Class<? extends BaseMetadataService> metaServiceClass, ClassLoader
metaServiceClassLoader) {
ClassLoader storeClassLoader =
Thread.currentThread().getContextClassLoader();
Thread.currentThread().setContextClassLoader(metaServiceClassLoader);
try {
@@ -73,7 +96,7 @@ public class MetadataUtils {
// Choose the first one
Constructor<?> constructor = acceptConstructor.get(0);
try {
- return (MetadataService) constructor.newInstance();
+ return (BaseMetadataService) constructor.newInstance();
} catch (Exception e) {
throw new MetaRuntimeException(
"Unable to construct meta service class: [" +
metaServiceClass.getName() + "]", e);
@@ -110,13 +133,14 @@ public class MetadataUtils {
return classNameList.toArray(new String[] {});
}
- public static Class<? extends MetadataService> loadMetaServiceClass(
+ public static Class<? extends BaseMetadataService> loadMetaServiceClass(
ClassLoader classLoader, String className, boolean initialize, String
notFoundMessage) {
// Try to load use expectClassName
try {
- return Class.forName(className, initialize,
classLoader).asSubclass(MetadataService.class);
+ return Class.forName(className, initialize, classLoader)
+ .asSubclass(BaseMetadataService.class);
} catch (ClassNotFoundException ne) {
- LOG.warn(notFoundMessage, ne);
+ LOG.warn(notFoundMessage);
}
return null;
}
@@ -172,6 +196,6 @@ public class MetadataUtils {
LOG.trace("Class: {} can not be found", className, t);
return false;
}
- return MetadataService.class.isAssignableFrom(clazz);
+ return BaseMetadataService.class.isAssignableFrom(clazz);
}
}
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/elasticsearch/src/main/java/org/apache/linkis/metadata/query/service/EsMetaService.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/elasticsearch/src/main/java/org/apache/linkis/metadata/query/service/EsMetaService.java
index 74bcbb192..b66a26f01 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/elasticsearch/src/main/java/org/apache/linkis/metadata/query/service/EsMetaService.java
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/elasticsearch/src/main/java/org/apache/linkis/metadata/query/service/EsMetaService.java
@@ -19,14 +19,14 @@ package org.apache.linkis.metadata.query.service;
import org.apache.linkis.datasourcemanager.common.util.json.Json;
import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo;
-import org.apache.linkis.metadata.query.common.service.AbstractMetaService;
+import org.apache.linkis.metadata.query.common.service.AbstractDbMetaService;
import org.apache.linkis.metadata.query.common.service.MetadataConnection;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
-public class EsMetaService extends AbstractMetaService<ElasticConnection> {
+public class EsMetaService extends AbstractDbMetaService<ElasticConnection> {
@Override
public MetadataConnection<ElasticConnection> getConnection(
String operator, Map<String, Object> params) throws Exception {
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/pom.xml
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/pom.xml
new file mode 100644
index 000000000..b040afeed
--- /dev/null
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/pom.xml
@@ -0,0 +1,110 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.linkis</groupId>
+ <artifactId>linkis</artifactId>
+ <version>1.3.0</version>
+ <relativePath>../../../../../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>linkis-metadata-query-service-hdfs</artifactId>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.linkis</groupId>
+ <artifactId>linkis-metadata-query-common</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.linkis</groupId>
+ <artifactId>linkis-module</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.linkis</groupId>
+ <artifactId>linkis-hadoop-common</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.linkis</groupId>
+ <artifactId>linkis-storage</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>3.2.0</version>
+ <inherited>false</inherited>
+ <configuration>
+ <skipAssembly>false</skipAssembly>
+ <finalName>out</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <attach>false</attach>
+ <descriptors>
+ <descriptor>src/main/assembly/distribution.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assembly/distribution.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/assembly/distribution.xml
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/assembly/distribution.xml
new file mode 100644
index 000000000..912fabc9a
--- /dev/null
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/assembly/distribution.xml
@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<assembly
+
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3
http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>linkis-metadata-query-service-hdfs</id>
+ <formats>
+ <format>zip</format>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <baseDirectory>linkis-metadata-query-service-es</baseDirectory>
+
+ <dependencySets>
+ <dependencySet>
+ <!-- Enable access to all projects in the current multimodule
build! <useAllReactorProjects>true</useAllReactorProjects> -->
+ <!-- Now, select which projects to include in this module-set. -->
+ <outputDirectory>lib</outputDirectory>
+ <useProjectArtifact>true</useProjectArtifact>
+ <useTransitiveDependencies>true</useTransitiveDependencies>
+ <unpack>false</unpack>
+ <useStrictFiltering>false</useStrictFiltering>
+ <useTransitiveFiltering>true</useTransitiveFiltering>
+ </dependencySet>
+ </dependencySets>
+
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/src/main/resources</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <fileMode>0777</fileMode>
+ <outputDirectory>conf</outputDirectory>
+ <lineEnding>unix</lineEnding>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsConnection.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsConnection.java
new file mode 100644
index 000000000..8bec4adb9
--- /dev/null
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsConnection.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.metadata.query.service;
+
+import org.apache.linkis.hadoop.common.utils.HDFSUtils;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Hdfs connection */
+public class HdfsConnection implements Closeable {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(HdfsConnection.class);
+
+ /** Hadoop configuration */
+ private final Configuration hadoopConf;
+
+ /** File system */
+ private final FileSystem fs;
+
+ public HdfsConnection(String scheme, String operator, String clusterLabel,
boolean cache)
+ throws IOException {
+ // TODO fix the problem of connecting multiple cluster in
FSFactory.getFSByLabelAndUser
+ // Fs fileSystem = FSFactory.getFSByLabelAndUser(scheme, operator,
clusterLabel);
+ hadoopConf = HDFSUtils.getConfigurationByLabel(operator, clusterLabel);
+ fs = createFileSystem(operator, this.hadoopConf, cache);
+ }
+
+ public HdfsConnection(
+ String scheme, String operator, Map<String, String> configuration,
boolean cache) {
+ if (Objects.nonNull(configuration)) {
+ hadoopConf = new Configuration();
+ configuration.forEach(hadoopConf::set);
+ } else {
+ hadoopConf = HDFSUtils.getConfiguration(operator);
+ }
+ fs = createFileSystem(operator, this.hadoopConf, cache);
+ }
+
+ @Override
+ public void close() throws IOException {
+ this.fs.close();
+ }
+
+ /**
+ * Get schema value
+ *
+ * @return schema
+ */
+ public String getSchema() {
+ return fs.getScheme();
+ }
+
+ /**
+ * Get hadoop configuration
+ *
+ * @return configuration
+ */
+ public Configuration getConfiguration() {
+ return this.hadoopConf;
+ }
+
+ /**
+ * Get file system
+ *
+ * @return file system
+ */
+ public FileSystem getFileSystem() {
+ return this.fs;
+ }
+ /**
+ * Create file system
+ *
+ * @param operator operator
+ * @param hadoopConf hadoop conf
+ * @param cache cache
+ * @return file system
+ */
+ private FileSystem createFileSystem(String operator, Configuration
hadoopConf, boolean cache) {
+ if (!cache) {
+ hadoopConf.set("fs.hdfs.impl.disable.cache", "true");
+ }
+ return HDFSUtils.createFileSystem(operator, hadoopConf);
+ }
+}
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsMetaService.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsMetaService.java
new file mode 100644
index 000000000..443685903
--- /dev/null
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsMetaService.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.metadata.query.service;
+
+import org.apache.linkis.common.conf.CommonVars;
+import org.apache.linkis.datasourcemanager.common.util.json.Json;
+import org.apache.linkis.hadoop.common.conf.HadoopConf;
+import org.apache.linkis.metadata.query.common.exception.MetaRuntimeException;
+import org.apache.linkis.metadata.query.common.service.AbstractFsMetaService;
+import org.apache.linkis.metadata.query.common.service.MetadataConnection;
+import org.apache.linkis.metadata.query.service.conf.ConfigurationUtils;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Hdfs meta service */
+public class HdfsMetaService extends AbstractFsMetaService<HdfsConnection> {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(HdfsMetaService.class);
+
+ private static final String PARAM_FILTER_RULE = "filter";
+ /** Filter rules */
+ private static final CommonVars<String> DEFAULT_FILTER_RULES =
+ CommonVars.apply(
+ "wds.linkis.server.mdm.service.hadoop.filter.rules",
+ StringUtils.join(
+ new String[] {
+ "fs.defaultFS",
+ "dfs.nameservices",
+ "dfs.ha.namenodes.<service>",
+ "dfs.namenode.rpc-address.<suffix>",
+ "dfs.client.failover.proxy.provider.<suffix>"
+ },
+ ","));
+
+ @Override
+ public MetadataConnection<HdfsConnection> getConnection(
+ String creator, Map<String, Object> params) throws Exception {
+ Map<String, String> hadoopConf = toMap(params,
HdfsParamsMapper.PARAM_HADOOP_CONF.getValue());
+ if (Objects.nonNull(hadoopConf) && !hadoopConf.isEmpty()) {
+ return new MetadataConnection<>(
+ new HdfsConnection("", creator, hadoopConf, !useCache()), true);
+ } else {
+ String clusterLabel =
+ Optional.ofNullable(toMap(params, "labels"))
+ .orElse(Collections.emptyMap())
+ .get(HdfsParamsMapper.PARAM_HADOOP_LABEL_CLUSTER.getValue());
+ LOG.info("Use Hadoop root config directory: " +
HadoopConf.hadoopConfDir());
+ return new MetadataConnection<>(
+ new HdfsConnection("", creator, clusterLabel, !useCache()), true);
+ }
+ }
+
+ @Override
+ public Map<String, String> queryConnectionInfo(
+ HdfsConnection connection, Map<String, String> queryParams) {
+ List<String> filterRules = new ArrayList<>();
+ AtomicReference<URI> uriReference = new AtomicReference<>();
+ Optional.ofNullable(queryParams.get("uri"))
+ .ifPresent(
+ uri -> {
+ try {
+ uriReference.set(new URI(uri));
+ } catch (URISyntaxException e) {
+ LOG.warn("Unrecognized uri value: [" + uri + "]", e);
+ }
+ });
+ Optional.ofNullable(queryParams.get(PARAM_FILTER_RULE))
+ .ifPresent(
+ rules -> {
+ if (StringUtils.isNotBlank(rules)) {
+ filterRules.addAll(Arrays.asList(rules.split(",")));
+ }
+ });
+ if (filterRules.isEmpty()) {
+
filterRules.addAll(Arrays.asList(DEFAULT_FILTER_RULES.getValue().split(",")));
+ }
+ return ConfigurationUtils.filterConfiguration(
+ connection.getFileSystem(), filterRules, uriReference.get());
+ }
+
+ @SuppressWarnings("unchecked")
+ private Map<String, String> toMap(Map<String, Object> connectParams, String
key) {
+ Map<String, String> valueMap = new HashMap<>();
+ Object mapObj = connectParams.get(key);
+ if (Objects.nonNull(mapObj)) {
+ try {
+ if (!(mapObj instanceof Map)) {
+ valueMap = Json.fromJson(String.valueOf(mapObj), Map.class,
String.class, String.class);
+ } else {
+ valueMap = (Map<String, String>) mapObj;
+ }
+ } catch (Exception e) {
+ throw new MetaRuntimeException("Cannot parse the param:[" + key + "]",
null);
+ }
+ }
+ return valueMap;
+ }
+}
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/BaseMetadataService.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsParamsMapper.java
similarity index 64%
copy from
linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/BaseMetadataService.java
copy to
linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsParamsMapper.java
index e2cf804b3..56e7efc66 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/service/BaseMetadataService.java
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsParamsMapper.java
@@ -15,19 +15,15 @@
* limitations under the License.
*/
-package org.apache.linkis.metadata.query.common.service;
+package org.apache.linkis.metadata.query.service;
-import java.io.Closeable;
-import java.util.Map;
+import org.apache.linkis.common.conf.CommonVars;
-public interface BaseMetadataService {
+public class HdfsParamsMapper {
- /**
- * Get connection
- *
- * @param params connect params
- * @return
- */
- MetadataConnection<? extends Closeable> getConnection(String operator,
Map<String, Object> params)
- throws Exception;
+ public static final CommonVars<String> PARAM_HADOOP_CONF =
+ CommonVars.apply("wds.linkis.server.mdm.service.hadoop.conf",
"hadoopConf");
+
+ public static final CommonVars<String> PARAM_HADOOP_LABEL_CLUSTER =
+ CommonVars.apply("wds.linkis.server.mdm.service.hadoop.label.cluster",
"hadoopCluster");
}
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/conf/ConfigurationUtils.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/conf/ConfigurationUtils.java
new file mode 100644
index 000000000..9a1d49c60
--- /dev/null
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/conf/ConfigurationUtils.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.metadata.query.service.conf;
+
+import org.apache.linkis.common.conf.CommonVars;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.viewfs.Constants;
+import org.apache.hadoop.fs.viewfs.ViewFileSystem;
+
+import java.net.URI;
+import java.util.*;
+import java.util.function.Function;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/** Utils to deal with configuration */
+public class ConfigurationUtils {
+
+ private static final CommonVars<String> CONFIG_VIEWFS_LINK_FALLBACK =
+ CommonVars.apply("wds.linkis.server.mdm.hadoop.conf.link.fallback",
"linkFallback");
+
+ private static final CommonVars<String> CONFIG_VIEWFS_LINK_NFLY =
+ CommonVars.apply("wds.linkis.server.mdm.hadoop.conf.link.nfly",
"linkNfly");
+ /** Placeholder */
+ private static final Pattern PLACEHOLDER_PATTERN =
Pattern.compile("<[^>]*?>");
+
+ private static final Function<String, Boolean> FILTER_ALL_CONFIG = config ->
false;
+
+ /**
+ * Filter configuration
+ *
+ * @param fileSystem file system
+ * @param filterRules filter rules
+ * @param uri uri
+ * @return props
+ */
+ public static Map<String, String> filterConfiguration(
+ FileSystem fileSystem, List<String> filterRules, URI uri) {
+ Map<String, String> filteredProps = new HashMap<>();
+ Configuration hadoopConf = fileSystem.getConf();
+ List<String> rules =
+ Objects.isNull(filterRules) ? new ArrayList<>() : new
ArrayList<>(filterRules);
+ Function<String, Boolean> acceptableFunction = FILTER_ALL_CONFIG;
+ if (fileSystem instanceof ViewFileSystem) {
+ acceptableFunction = addViewFileSystemFilterRules(fileSystem, rules,
uri);
+ }
+ Pattern pattern = rulesToPattern(rules);
+ Function<String, Boolean> finalAcceptableFunction = acceptableFunction;
+ hadoopConf.forEach(
+ entry -> {
+ String key = entry.getKey();
+ if (pattern.matcher(key).matches() ||
finalAcceptableFunction.apply(key)) {
+ filteredProps.put(key, entry.getValue());
+ }
+ });
+ return filteredProps;
+ }
+
+ /**
+ * Filter rules to pattern
+ *
+ * @param filterRules filter rules
+ * @return pattern
+ */
+ private static Pattern rulesToPattern(List<String> filterRules) {
+ StringBuffer sb = new StringBuffer("^(");
+ for (int i = 0; i < filterRules.size(); i++) {
+ String rule = filterRules.get(i);
+ if (StringUtils.isNotBlank(rule)) {
+ Matcher matcher = PLACEHOLDER_PATTERN.matcher(rule);
+ while (matcher.find()) {
+ matcher.appendReplacement(sb, "[\\\\s\\\\S]*?");
+ }
+ matcher.appendTail(sb);
+ if (i < filterRules.size() - 1) {
+ sb.append("|");
+ }
+ }
+ }
+ ;
+ sb.append(")$");
+ return Pattern.compile(sb.toString().replace(".", "[.]"));
+ }
+
+ /**
+ * Add filter rules for view FileSystem
+ *
+ * @param filerRules filter rules
+ * @param uri uri
+ * @return filter function
+ */
+ private static Function<String, Boolean> addViewFileSystemFilterRules(
+ FileSystem fileSystem, List<String> filerRules, URI uri) {
+ String mountTableName =
+ Optional.ofNullable(uri)
+ .orElse(FileSystem.getDefaultUri(fileSystem.getConf()))
+ .getAuthority();
+ if (StringUtils.isBlank(mountTableName)) {
+ mountTableName = Constants.CONFIG_VIEWFS_DEFAULT_MOUNT_TABLE;
+ }
+ if (Objects.nonNull(uri) &&
fileSystem.getScheme().equals(uri.getScheme())) {
+ // Just load the default mountable configuration
+ String linkPrefix = Constants.CONFIG_VIEWFS_PREFIX + "." +
mountTableName + ".";
+ String linkBasicPrefix = linkPrefix + Constants.CONFIG_VIEWFS_LINK;
+ String linkMergePrefix = linkPrefix + Constants.CONFIG_VIEWFS_LINK_MERGE;
+ // linkFallback, linkNfly in HADOOP-13055
+ filerRules.add(linkPrefix + Constants.CONFIG_VIEWFS_LINK_MERGE_SLASH);
+ filerRules.add(linkPrefix + CONFIG_VIEWFS_LINK_FALLBACK.getValue());
+ filerRules.add(linkPrefix + CONFIG_VIEWFS_LINK_NFLY.getValue());
+ filerRules.add(linkPrefix + Constants.CONFIG_VIEWFS_HOMEDIR);
+ String path = uri.getPath();
+ return config ->
+ (config.startsWith(linkBasicPrefix)
+ && path.startsWith(config.substring(linkBasicPrefix.length()
+ 1)))
+ || (config.startsWith(linkMergePrefix)
+ && path.startsWith(config.substring(linkMergePrefix.length()
+ 1)));
+ } else {
+ // Load in all the mountable configuration
+ filerRules.add(Constants.CONFIG_VIEWFS_PREFIX + "." + mountTableName +
".<suffix>");
+ }
+ return FILTER_ALL_CONFIG;
+ }
+}
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/src/main/java/org/apache/linkis/metadata/query/service/HiveMetaService.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/src/main/java/org/apache/linkis/metadata/query/service/HiveMetaService.java
index 5a506373f..4fc85e629 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/src/main/java/org/apache/linkis/metadata/query/service/HiveMetaService.java
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/src/main/java/org/apache/linkis/metadata/query/service/HiveMetaService.java
@@ -25,7 +25,7 @@ import
org.apache.linkis.datasourcemanager.common.util.json.Json;
import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo;
import org.apache.linkis.metadata.query.common.domain.MetaPartitionInfo;
import org.apache.linkis.metadata.query.common.exception.MetaRuntimeException;
-import org.apache.linkis.metadata.query.common.service.AbstractMetaService;
+import org.apache.linkis.metadata.query.common.service.AbstractDbMetaService;
import org.apache.linkis.metadata.query.common.service.MetadataConnection;
import org.apache.commons.io.FileUtils;
@@ -46,7 +46,7 @@ import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class HiveMetaService extends AbstractMetaService<HiveConnection> {
+public class HiveMetaService extends AbstractDbMetaService<HiveConnection> {
private static final Logger LOG =
LoggerFactory.getLogger(HiveMetaService.class);
private static final CommonVars<String> TMP_FILE_STORE_LOCATION =
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/kafka/src/main/java/org/apache/linkis/metadata/query/service/KafkaMetaService.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/kafka/src/main/java/org/apache/linkis/metadata/query/service/KafkaMetaService.java
index a282ecb61..d1e30ac88 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/kafka/src/main/java/org/apache/linkis/metadata/query/service/KafkaMetaService.java
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/kafka/src/main/java/org/apache/linkis/metadata/query/service/KafkaMetaService.java
@@ -22,7 +22,7 @@ import org.apache.linkis.bml.client.BmlClientFactory;
import org.apache.linkis.bml.protocol.BmlDownloadResponse;
import org.apache.linkis.common.conf.CommonVars;
import org.apache.linkis.metadata.query.common.exception.MetaRuntimeException;
-import org.apache.linkis.metadata.query.common.service.AbstractMetaService;
+import org.apache.linkis.metadata.query.common.service.AbstractDbMetaService;
import org.apache.linkis.metadata.query.common.service.MetadataConnection;
import org.apache.commons.io.FileUtils;
@@ -44,7 +44,7 @@ import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class KafkaMetaService extends AbstractMetaService<KafkaConnection> {
+public class KafkaMetaService extends AbstractDbMetaService<KafkaConnection> {
private static final Logger LOG =
LoggerFactory.getLogger(KafkaMetaService.class);
private static final CommonVars<String> TMP_FILE_STORE_LOCATION =
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/Db2MetaService.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/Db2MetaService.java
index f92cb8738..810899303 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/Db2MetaService.java
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/Db2MetaService.java
@@ -19,7 +19,7 @@ package org.apache.linkis.metadata.query.service;
import org.apache.linkis.datasourcemanager.common.util.json.Json;
import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo;
-import org.apache.linkis.metadata.query.common.service.AbstractMetaService;
+import org.apache.linkis.metadata.query.common.service.AbstractDbMetaService;
import org.apache.linkis.metadata.query.common.service.MetadataConnection;
import org.apache.linkis.metadata.query.service.conf.SqlParamsMapper;
import org.apache.linkis.metadata.query.service.db2.SqlConnection;
@@ -32,7 +32,7 @@ import java.util.List;
import java.util.Map;
@Component
-public class Db2MetaService extends AbstractMetaService<SqlConnection> {
+public class Db2MetaService extends AbstractDbMetaService<SqlConnection> {
@Override
public MetadataConnection<SqlConnection> getConnection(
String operator, Map<String, Object> params) throws Exception {
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/DmMetaService.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/DmMetaService.java
index 464f37a85..eedc65080 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/DmMetaService.java
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/DmMetaService.java
@@ -19,7 +19,7 @@ package org.apache.linkis.metadata.query.service;
import org.apache.linkis.datasourcemanager.common.util.json.Json;
import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo;
-import org.apache.linkis.metadata.query.common.service.AbstractMetaService;
+import org.apache.linkis.metadata.query.common.service.AbstractDbMetaService;
import org.apache.linkis.metadata.query.common.service.MetadataConnection;
import org.apache.linkis.metadata.query.service.conf.SqlParamsMapper;
import org.apache.linkis.metadata.query.service.dm.SqlConnection;
@@ -29,7 +29,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-public class DmMetaService extends AbstractMetaService<SqlConnection> {
+public class DmMetaService extends AbstractDbMetaService<SqlConnection> {
@Override
public MetadataConnection<SqlConnection> getConnection(
String operator, Map<String, Object> params) throws Exception {
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/GreenplumMetaService.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/GreenplumMetaService.java
index f882d2a71..646ea73ad 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/GreenplumMetaService.java
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/GreenplumMetaService.java
@@ -19,7 +19,7 @@ package org.apache.linkis.metadata.query.service;
import org.apache.linkis.datasourcemanager.common.util.json.Json;
import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo;
-import org.apache.linkis.metadata.query.common.service.AbstractMetaService;
+import org.apache.linkis.metadata.query.common.service.AbstractDbMetaService;
import org.apache.linkis.metadata.query.common.service.MetadataConnection;
import org.apache.linkis.metadata.query.service.conf.SqlParamsMapper;
import org.apache.linkis.metadata.query.service.greenplum.SqlConnection;
@@ -31,7 +31,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-public class GreenplumMetaService extends AbstractMetaService<SqlConnection> {
+public class GreenplumMetaService extends AbstractDbMetaService<SqlConnection>
{
@Override
public MetadataConnection<SqlConnection> getConnection(
String operator, Map<String, Object> params) throws Exception {
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/KingbaseMetaService.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/KingbaseMetaService.java
index c65d846e8..4e4fe4190 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/KingbaseMetaService.java
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/KingbaseMetaService.java
@@ -19,7 +19,7 @@ package org.apache.linkis.metadata.query.service;
import org.apache.linkis.datasourcemanager.common.util.json.Json;
import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo;
-import org.apache.linkis.metadata.query.common.service.AbstractMetaService;
+import org.apache.linkis.metadata.query.common.service.AbstractDbMetaService;
import org.apache.linkis.metadata.query.common.service.MetadataConnection;
import org.apache.linkis.metadata.query.service.conf.SqlParamsMapper;
import org.apache.linkis.metadata.query.service.kingbase.SqlConnection;
@@ -29,7 +29,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-public class KingbaseMetaService extends AbstractMetaService<SqlConnection> {
+public class KingbaseMetaService extends AbstractDbMetaService<SqlConnection> {
@Override
public MetadataConnection<SqlConnection> getConnection(
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/MysqlMetaService.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/MysqlMetaService.java
index 7b125bae4..325a8d0fb 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/MysqlMetaService.java
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/MysqlMetaService.java
@@ -19,7 +19,7 @@ package org.apache.linkis.metadata.query.service;
import org.apache.linkis.datasourcemanager.common.util.json.Json;
import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo;
-import org.apache.linkis.metadata.query.common.service.AbstractMetaService;
+import org.apache.linkis.metadata.query.common.service.AbstractDbMetaService;
import org.apache.linkis.metadata.query.common.service.MetadataConnection;
import org.apache.linkis.metadata.query.service.conf.SqlParamsMapper;
import org.apache.linkis.metadata.query.service.mysql.SqlConnection;
@@ -31,7 +31,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-public class MysqlMetaService extends AbstractMetaService<SqlConnection> {
+public class MysqlMetaService extends AbstractDbMetaService<SqlConnection> {
@Override
public MetadataConnection<SqlConnection> getConnection(
String operator, Map<String, Object> params) throws Exception {
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/OracleMetaService.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/OracleMetaService.java
index 15b36d7ab..2d23542bd 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/OracleMetaService.java
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/OracleMetaService.java
@@ -19,7 +19,7 @@ package org.apache.linkis.metadata.query.service;
import org.apache.linkis.datasourcemanager.common.util.json.Json;
import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo;
-import org.apache.linkis.metadata.query.common.service.AbstractMetaService;
+import org.apache.linkis.metadata.query.common.service.AbstractDbMetaService;
import org.apache.linkis.metadata.query.common.service.MetadataConnection;
import org.apache.linkis.metadata.query.service.conf.SqlParamsMapper;
import org.apache.linkis.metadata.query.service.oracle.SqlConnection;
@@ -32,7 +32,7 @@ import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class OracleMetaService extends AbstractMetaService<SqlConnection> {
+public class OracleMetaService extends AbstractDbMetaService<SqlConnection> {
private static final Logger LOG =
LoggerFactory.getLogger(OracleMetaService.class);
@Override
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/PostgresqlMetaService.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/PostgresqlMetaService.java
index 7dbe633e7..134ddb341 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/PostgresqlMetaService.java
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/PostgresqlMetaService.java
@@ -19,7 +19,7 @@ package org.apache.linkis.metadata.query.service;
import org.apache.linkis.datasourcemanager.common.util.json.Json;
import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo;
-import org.apache.linkis.metadata.query.common.service.AbstractMetaService;
+import org.apache.linkis.metadata.query.common.service.AbstractDbMetaService;
import org.apache.linkis.metadata.query.common.service.MetadataConnection;
import org.apache.linkis.metadata.query.service.conf.SqlParamsMapper;
import org.apache.linkis.metadata.query.service.postgres.SqlConnection;
@@ -31,7 +31,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-public class PostgresqlMetaService extends AbstractMetaService<SqlConnection> {
+public class PostgresqlMetaService extends
AbstractDbMetaService<SqlConnection> {
@Override
public MetadataConnection<SqlConnection> getConnection(
String operator, Map<String, Object> params) throws Exception {
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/SqlserverMetaService.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/SqlserverMetaService.java
index 9225a0d0b..3d9b48742 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/SqlserverMetaService.java
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mysql/src/main/java/org/apache/linkis/metadata/query/service/SqlserverMetaService.java
@@ -19,7 +19,7 @@ package org.apache.linkis.metadata.query.service;
import org.apache.linkis.datasourcemanager.common.util.json.Json;
import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo;
-import org.apache.linkis.metadata.query.common.service.AbstractMetaService;
+import org.apache.linkis.metadata.query.common.service.AbstractDbMetaService;
import org.apache.linkis.metadata.query.common.service.MetadataConnection;
import org.apache.linkis.metadata.query.service.conf.SqlParamsMapper;
import org.apache.linkis.metadata.query.service.sqlserver.SqlConnection;
@@ -29,7 +29,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-public class SqlserverMetaService extends AbstractMetaService<SqlConnection> {
+public class SqlserverMetaService extends AbstractDbMetaService<SqlConnection>
{
@Override
public MetadataConnection<SqlConnection> getConnection(
String operator, Map<String, Object> params) throws Exception {
diff --git a/linkis-public-enhancements/linkis-datasource/pom.xml
b/linkis-public-enhancements/linkis-datasource/pom.xml
index 531c903a8..e2db91bb4 100644
--- a/linkis-public-enhancements/linkis-datasource/pom.xml
+++ b/linkis-public-enhancements/linkis-datasource/pom.xml
@@ -34,6 +34,7 @@
<module>linkis-metadata-query/service/hive</module>
<module>linkis-metadata-query/service/kafka</module>
<module>linkis-metadata-query/service/mysql</module>
+ <module>linkis-metadata-query/service/hdfs</module>
<module>linkis-metadata-query/server</module>
<module>linkis-datasource-client</module>
</modules>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]