This is an automated email from the ASF dual-hosted git repository.
zongwen pushed a commit to branch cdc-multiple-table
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/cdc-multiple-table by this
push:
new d144e176b [improve][api] Support for getting tables from the catalog
(#4170)
d144e176b is described below
commit d144e176b3f69f6f3d0d84607d00263555910962
Author: Zongwen Li <[email protected]>
AuthorDate: Mon Feb 20 18:50:31 2023 +0800
[improve][api] Support for getting tables from the catalog (#4170)
---
.../apache/seatunnel/api/common/CommonOptions.java | 35 +++++++++++
.../api/configuration/util/ConfigUtil.java | 5 +-
.../api/table/catalog/CatalogOptions.java | 68 ++++++++++++++++++++++
.../api/table/catalog/CatalogTableUtil.java | 61 +++++++++++++++++++
4 files changed, 167 insertions(+), 2 deletions(-)
diff --git
a/seatunnel-api/src/main/java/org/apache/seatunnel/api/common/CommonOptions.java
b/seatunnel-api/src/main/java/org/apache/seatunnel/api/common/CommonOptions.java
new file mode 100644
index 000000000..18f46873f
--- /dev/null
+++
b/seatunnel-api/src/main/java/org/apache/seatunnel/api/common/CommonOptions.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.common;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+
+public interface CommonOptions {
+ Option<String> FACTORY_ID =
+ Options.key("factory")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Identifier of the SPI factory class.");
+
+ Option<String> PLUGIN_NAME =
+ Options.key("plugin_name")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Name of the SPI plugin class.");
+}
diff --git
a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConfigUtil.java
b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConfigUtil.java
index 40ddbf849..b65103bd5 100644
---
a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConfigUtil.java
+++
b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConfigUtil.java
@@ -26,7 +26,7 @@ import
org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -43,6 +43,7 @@ public class ConfigUtil {
* </pre>
*/
public static Map<String, Object> treeMap(Object rawMap) {
+ // TODO: Keeping the order of the values in the map
try {
return
PROPERTIES_MAPPER.readValue(PROPERTIES_MAPPER.writeValueAsString(rawMap), new
TypeReference<Map<String, Object>>() {
});
@@ -76,7 +77,7 @@ public class ConfigUtil {
Map<String, Object> rawMap = (Map<String, Object>) rawValue;
if (!nestedMap) {
keys = new ArrayList<>();
- newMap = new HashMap<>(rawMap.size());
+ newMap = new LinkedHashMap<>(rawMap.size());
}
for (Map.Entry<String, Object> entry : rawMap.entrySet()) {
keys.add(entry.getKey());
diff --git
a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/CatalogOptions.java
b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/CatalogOptions.java
new file mode 100644
index 000000000..fc4e4a237
--- /dev/null
+++
b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/CatalogOptions.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.table.catalog;
+
+import org.apache.seatunnel.api.common.CommonOptions;
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+
+import java.util.List;
+import java.util.Map;
+
+public interface CatalogOptions {
+ Option<Map<String, String>> CATALOG_OPTIONS =
+ Options.key("catalog")
+ .mapType()
+ .noDefaultValue()
+ .withDescription("configuration options for the catalog.");
+
+ Option<String> NAME =
+ Options.key("name")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("catalog name");
+
+ Option<List<String>> TABLE_NAMES =
+ Options.key("table-names")
+ .listType()
+ .noDefaultValue()
+ .withDescription("List of table names of databases to capture." +
+ "The table name needs to include the database name, for
example: database_name.table_name");
+
+ Option<String> DATABASE_PATTERN =
+ Options.key("database-pattern")
+ .stringType()
+ .defaultValue(".*")
+ .withDescription("The database names RegEx of the database to
capture.");
+
+ Option<String> TABLE_PATTERN =
+ Options.key("table-pattern")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("The table names RegEx of the database to
capture." +
+ "The table name needs to include the database name, for
example: database_.*\\.table_.*");
+
+ OptionRule.Builder BASE_RULE =
+ OptionRule.builder()
+ .optional(CommonOptions.FACTORY_ID)
+ .optional(NAME)
+ .optional(DATABASE_PATTERN)
+ .exclusive(TABLE_PATTERN, TABLE_NAMES);
+
+}
diff --git
a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/CatalogTableUtil.java
b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/CatalogTableUtil.java
index 55bb14101..d4ed86d4e 100644
---
a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/CatalogTableUtil.java
+++
b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/CatalogTableUtil.java
@@ -17,8 +17,12 @@
package org.apache.seatunnel.api.table.catalog;
+import org.apache.seatunnel.api.common.CommonOptions;
import org.apache.seatunnel.api.configuration.Option;
import org.apache.seatunnel.api.configuration.Options;
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.api.table.factory.FactoryException;
+import org.apache.seatunnel.api.table.factory.FactoryUtil;
import org.apache.seatunnel.api.table.type.ArrayType;
import org.apache.seatunnel.api.table.type.BasicType;
import org.apache.seatunnel.api.table.type.DecimalType;
@@ -39,13 +43,16 @@ import
org.apache.seatunnel.shade.com.typesafe.config.Config;
import org.apache.seatunnel.shade.com.typesafe.config.ConfigRenderOptions;
import lombok.Getter;
+import org.apache.commons.lang3.StringUtils;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.regex.Pattern;
public class CatalogTableUtil implements Serializable {
public static final Option<Map<String, String>> SCHEMA =
Options.key("schema").mapType().noDefaultValue().withDescription("SeaTunnel
Schema");
@@ -62,6 +69,60 @@ public class CatalogTableUtil implements Serializable {
this.catalogTable = catalogTable;
}
+ private static List<CatalogTable> getCatalogTables(Config config,
ClassLoader classLoader) {
+ ReadonlyConfig readonlyConfig = ReadonlyConfig.fromConfig(config);
+ Map<String, String> catalogOptions =
readonlyConfig.getOptional(CatalogOptions.CATALOG_OPTIONS).orElse(new
HashMap<>());
+ // TODO: fallback key
+ String factoryId =
catalogOptions.getOrDefault(CommonOptions.FACTORY_ID.key(),
readonlyConfig.get(CommonOptions.PLUGIN_NAME));
+ Map<String, Object> catalogAllOptions = new HashMap<>();
+ catalogAllOptions.putAll(readonlyConfig.toMap());
+ catalogAllOptions.putAll(catalogOptions);
+ ReadonlyConfig catalogConfig =
ReadonlyConfig.fromMap(catalogAllOptions);
+
+ // Highest priority: specified schema
+ Map<String, String> schemaMap =
readonlyConfig.get(CatalogTableUtil.SCHEMA);
+ if (schemaMap != null && schemaMap.size() > 0) {
+ CatalogTable catalogTable =
CatalogTableUtil.buildWithConfig(config).getCatalogTable();
+ return Collections.singletonList(catalogTable);
+ }
+
+ Catalog catalog = null;
+ try {
+ catalog =
FactoryUtil.createCatalog(catalogConfig.get(CatalogOptions.NAME),
catalogConfig, classLoader, factoryId);
+ } catch (FactoryException e) {
+ return Collections.emptyList();
+ }
+
+ // Get the list of specified tables
+ List<String> tableNames =
catalogConfig.get(CatalogOptions.TABLE_NAMES);
+ List<CatalogTable> catalogTables = new ArrayList<>();
+ if (tableNames != null && tableNames.size() > 1) {
+ for (String tableName : tableNames) {
+ catalogTables.add(catalog.getTable(TablePath.of(tableName)));
+ }
+ return catalogTables;
+ }
+
+ // Get the list of table pattern
+ String tablePatternStr =
catalogConfig.get(CatalogOptions.TABLE_PATTERN);
+ if (StringUtils.isBlank(tablePatternStr)) {
+ return Collections.emptyList();
+ }
+ Pattern databasePattern =
Pattern.compile(catalogConfig.get(CatalogOptions.DATABASE_PATTERN));
+ Pattern tablePattern =
Pattern.compile(catalogConfig.get(CatalogOptions.TABLE_PATTERN));
+ List<String> allDatabase = catalog.listDatabases();
+ allDatabase.removeIf(s -> !databasePattern.matcher(s).matches());
+ for (String databaseName : allDatabase) {
+ tableNames = catalog.listTables(databaseName);
+ for (String tableName : tableNames) {
+ if (tablePattern.matcher(databaseName + "." +
tableName).matches()) {
+
catalogTables.add(catalog.getTable(TablePath.of(databaseName, tableName)));
+ }
+ }
+ }
+ return catalogTables;
+ }
+
public static CatalogTableUtil buildWithConfig(Config config) {
CheckResult checkResult = CheckConfigUtil.checkAllExists(config,
"schema");
if (!checkResult.isSuccess()) {
