This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-12970-8a81d13ea9c34610c430b5d47695e08af4091b34 in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit 8964879f41468951f4cca86705fa1b642bae2b00 Author: Daniel <[email protected]> AuthorDate: Tue Nov 17 15:31:13 2026 +0000 [Fix][Connector-V2] Fix DB2 CDC capture and savepoint recovery (#12970) Co-authored-by: davidzollo <[email protected]> --- docs/en/connectors/source/DB2-CDC.md | 20 ++ docs/zh/connectors/source/DB2-CDC.md | 14 + .../external/JdbcSourceFetchTaskContext.java | 6 +- .../cdc/debezium/ConnectTableChangeSerializer.java | 6 +- .../cdc/db2/config/Db2SourceConfigFactory.java | 54 ++-- .../seatunnel/cdc/db2/source/Db2Dialect.java | 22 +- .../db2/source/Db2IncrementalSourceFactory.java | 118 +++++++- .../reader/fetch/Db2SourceFetchTaskContext.java | 28 +- .../seatunnel/cdc/db2/utils/Db2CdcConnection.java | 72 +++++ .../cdc/db2/utils/Db2ConnectionUtils.java | 2 +- .../cdc/db2/utils/TableDiscoveryUtils.java | 46 ++- .../seatunnel/cdc/db2/source/Db2DialectTest.java | 91 ++++++ .../source/Db2IncrementalSourceFactoryTest.java | 154 ++++++++++ .../db2/source/Db2SavepointHistoryRestoreTest.java | 192 +++++++++++++ .../db2/source/Db2SourceFetchTaskContextTest.java | 178 ++++++++++++ .../cdc/db2/utils/Db2CdcConnectionTest.java | 104 +++++++ .../cdc/db2/utils/TableDiscoveryUtilsTest.java | 98 +++++++ .../source/split/VitessTableSchemaStateTest.java | 153 ++++++++++ .../seatunnel/e2e/connector/cdc/db2/Db2CDCIT.java | 320 ++++++++++++++++++++- .../db2cdc_to_db2_add_table_after_savepoint.conf | 51 ++++ .../db2cdc_to_db2_with_lower_case_table_names.conf | 51 ++++ .../resources/db2cdc_to_db2_with_savepoint.conf | 51 ++++ 22 files changed, 1771 insertions(+), 60 deletions(-) diff --git a/docs/en/connectors/source/DB2-CDC.md b/docs/en/connectors/source/DB2-CDC.md index 8eb7238e04..95864b5452 100644 --- a/docs/en/connectors/source/DB2-CDC.md +++ b/docs/en/connectors/source/DB2-CDC.md @@ -195,6 +195,26 @@ source { } ``` + +### Capture registrations and recovery + +Active Capture registrations remain readable as `CD_NEW_SYNCHPOINT` advances. +`CD_OLD_SYNCHPOINT` is the readable starting point, not a stop watermark. +The schema and table components in `table-names` and `table-names-config.table` are resolved +against the DB2 catalog before catalog lookup and capture filtering. If an object with exactly the +configured spelling exists, that spelling is used, so existing jobs keep selecting the same table +even when an object with the uppercase spelling also exists. Otherwise the components are +normalized to uppercase, the spelling DB2 gives unquoted identifiers. To select a specific object, +write its schema and table names exactly as stored in the DB2 catalog (`SYSCAT.TABLES`). Double +quotes are not interpreted in these options: a name that contains a quote character is used as +written, including the quotes, and does not match the object. The database component retains its +configured spelling, and `table-pattern` is kept unchanged. + +Savepoint recovery must preserve the schema component of DB2's two-part table identifiers. +A restored job can catch up changes committed while it was stopped and discover a newly +configured capture-enabled table. Existing configuration keys, offsets and checkpoint +serialization formats remain unchanged. + ## Changelog <ChangeLog /> diff --git a/docs/zh/connectors/source/DB2-CDC.md b/docs/zh/connectors/source/DB2-CDC.md index aa720a889a..796d4074b3 100644 --- a/docs/zh/connectors/source/DB2-CDC.md +++ b/docs/zh/connectors/source/DB2-CDC.md @@ -193,6 +193,20 @@ source { } ``` +### Capture 注册与恢复 + +活动 Capture 注册在 `CD_NEW_SYNCHPOINT` 推进后继续读取增量。 +`CD_OLD_SYNCHPOINT` 表示可读取的起点,不表示停止位点。 +`table-names` 和 `table-names-config.table` 中的 schema 和表名会在 catalog 查询与 Capture 过滤前 +先按 DB2 catalog 解析:若存在与配置拼写完全一致的对象,则使用该拼写,即使同时存在大写同名对象, +已有任务也会继续选择原来的表;否则将其转为大写,即 DB2 对不带引号标识符的存储拼写。需要选择特定对象时, +请按 DB2 catalog(`SYSCAT.TABLES`)中存储的拼写填写 schema 和表名。这些配置项不解析双引号: +包含引号字符的名称会连同引号原样使用,无法匹配对应对象。 +数据库名保留配置中的拼写,`table-pattern` 正则表达式保持原样。 + +保存点恢复应保留 DB2 两段表标识中的 schema。恢复后的任务可以追平停机期间的变化, +并发现新配置且已启用 Capture 的表。既有配置项、offset 和 checkpoint 序列化格式保持不变。 + ## 变更日志 <ChangeLog /> diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/reader/external/JdbcSourceFetchTaskContext.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/reader/external/JdbcSourceFetchTaskContext.java index a6c8f5588f..e8f0d95938 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/reader/external/JdbcSourceFetchTaskContext.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/reader/external/JdbcSourceFetchTaskContext.java @@ -188,9 +188,13 @@ public abstract class JdbcSourceFetchTaskContext implements FetchTask.Context { jsonConverter.toConnectData("topic", historyTableChanges.get(tableId)); Struct deserializedStruct = (Struct) schemaAndValue.value(); + // The split key preserves the namespace even when history has only two parts. + boolean useCatalogBeforeSchema = + tableId.catalog() != null && !tableId.catalog().isEmpty(); TableChanges tableChanges = tableChangeSerializer.deserialize( - Collections.singletonList(deserializedStruct), false); + Collections.singletonList(deserializedStruct), + useCatalogBeforeSchema); Iterator<TableChanges.TableChange> iterator = tableChanges.iterator(); TableChanges.TableChange tableChange = null; diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/debezium/ConnectTableChangeSerializer.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/debezium/ConnectTableChangeSerializer.java index c933e82a48..673697376a 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/debezium/ConnectTableChangeSerializer.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/debezium/ConnectTableChangeSerializer.java @@ -119,7 +119,11 @@ public class ConnectTableChangeSerializer String tableId = struct.getString(ID_KEY); TableChanges.TableChangeType changeType = TableChanges.TableChangeType.valueOf(struct.getString(TYPE_KEY)); - Table table = toTable(struct.getStruct(TABLE_KEY), TableId.parse(tableId)); + // Two-part identifiers mean catalog.table or schema.table depending on the caller. + Table table = + toTable( + struct.getStruct(TABLE_KEY), + TableId.parse(tableId, useCatalogBeforeSchema)); switch (changeType) { case CREATE: tableChanges.create(table); diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/config/Db2SourceConfigFactory.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/config/Db2SourceConfigFactory.java index 0c5caba483..a0d6cf20b7 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/config/Db2SourceConfigFactory.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/config/Db2SourceConfigFactory.java @@ -77,30 +77,34 @@ public class Db2SourceConfigFactory extends JdbcSourceConfigFactory { dbzProperties.forEach(props::put); } - return new Db2SourceConfig( - startupConfig, - stopConfig, - databaseList, - tableList, - splitSize, - splitColumn, - distributionFactorUpper, - distributionFactorLower, - sampleShardingThreshold, - inverseSamplingRate, - sampleShardingAllow, - props, - DRIVER_CLASS_NAME, - hostname, - port, - username, - password, - originUrl, - fetchSize, - serverTimeZone, - connectTimeoutMillis, - connectMaxRetries, - connectionPoolSize, - exactlyOnce); + Db2SourceConfig config = + new Db2SourceConfig( + startupConfig, + stopConfig, + databaseList, + tableList, + splitSize, + splitColumn, + distributionFactorUpper, + distributionFactorLower, + sampleShardingThreshold, + inverseSamplingRate, + sampleShardingAllow, + props, + DRIVER_CLASS_NAME, + hostname, + port, + username, + password, + originUrl, + fetchSize, + serverTimeZone, + connectTimeoutMillis, + connectMaxRetries, + connectionPoolSize, + exactlyOnce); + // Preserve the user-selected snapshot splitting mode in each reader configuration. + config.setEnableConcurrentRead(enableConcurrentRead); + return config; } } diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2Dialect.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2Dialect.java index 107feac297..26fbc3ff4e 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2Dialect.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2Dialect.java @@ -44,8 +44,10 @@ import io.debezium.connector.db2.Db2Connection; import io.debezium.jdbc.JdbcConnection; import io.debezium.relational.TableId; import io.debezium.relational.history.TableChanges; +import lombok.extern.slf4j.Slf4j; import java.sql.SQLException; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -53,6 +55,7 @@ import java.util.Optional; import java.util.Set; /** The {@link JdbcDataSourceDialect} implementation for Db2 datasource. */ +@Slf4j public class Db2Dialect implements JdbcDataSourceDialect { private static final long serialVersionUID = 1L; @@ -160,13 +163,26 @@ public class Db2Dialect implements JdbcDataSourceDialect { @Override public Optional<PrimaryKey> getPrimaryKey(JdbcConnection jdbcConnection, TableId tableId) { - return Optional.ofNullable( - tableMap.get(toDb2TableId(tableId)).getTableSchema().getPrimaryKey()); + CatalogTable table = tableMap.get(toDb2TableId(tableId)); + if (table == null) { + log.warn( + "Catalog metadata not found for DB2 table {}; primary key is unavailable", + tableId); + return Optional.empty(); + } + return Optional.ofNullable(table.getTableSchema().getPrimaryKey()); } @Override public List<ConstraintKey> getConstraintKeys(JdbcConnection jdbcConnection, TableId tableId) { - return tableMap.get(toDb2TableId(tableId)).getTableSchema().getConstraintKeys(); + CatalogTable table = tableMap.get(toDb2TableId(tableId)); + if (table == null) { + log.warn( + "Catalog metadata not found for DB2 table {}; constraint keys are unavailable", + tableId); + return Collections.emptyList(); + } + return table.getTableSchema().getConstraintKeys(); } private static Map<TableId, CatalogTable> createDb2TableMap(List<CatalogTable> catalogTables) { diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2IncrementalSourceFactory.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2IncrementalSourceFactory.java index 51f4490600..98761c56ef 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2IncrementalSourceFactory.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2IncrementalSourceFactory.java @@ -17,15 +17,18 @@ package org.apache.seatunnel.connectors.seatunnel.cdc.db2.source; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.options.ConnectorCommonOptions; import org.apache.seatunnel.api.source.SeaTunnelSource; import org.apache.seatunnel.api.source.SourceSplit; +import org.apache.seatunnel.api.table.catalog.Catalog; import org.apache.seatunnel.api.table.catalog.CatalogTable; import org.apache.seatunnel.api.table.catalog.CatalogTableUtil; import org.apache.seatunnel.api.table.catalog.TablePath; import org.apache.seatunnel.api.table.connector.TableSource; import org.apache.seatunnel.api.table.factory.Factory; +import org.apache.seatunnel.api.table.factory.FactoryUtil; import org.apache.seatunnel.api.table.factory.TableSourceFactory; import org.apache.seatunnel.api.table.factory.TableSourceFactoryContext; import org.apache.seatunnel.connectors.cdc.base.config.JdbcSourceTableConfig; @@ -33,13 +36,23 @@ import org.apache.seatunnel.connectors.cdc.base.option.SourceOptions; import org.apache.seatunnel.connectors.cdc.base.option.StartupMode; import org.apache.seatunnel.connectors.cdc.base.option.StopMode; import org.apache.seatunnel.connectors.cdc.base.utils.CatalogTableUtils; +import org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils.TableDiscoveryUtils; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.DatabaseIdentifier; import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.apache.seatunnel.api.configuration.util.Conditions.notBlank; @@ -114,12 +127,12 @@ public class Db2IncrementalSourceFactory implements TableSourceFactory { } catch (Exception e) { log.warn("Failed to load JDBC driver {}", DRIVER_CLASS_NAME, e); } + ReadonlyConfig options = + resolveDb2Options(context.getOptions(), context.getClassLoader()); List<CatalogTable> catalogTables = - CatalogTableUtil.getCatalogTables( - context.getOptions(), context.getClassLoader()); + CatalogTableUtil.getCatalogTables(options, context.getClassLoader()); Optional<List<JdbcSourceTableConfig>> tableConfigs = - context.getOptions() - .getOptional(Db2IncrementalSourceOptions.TABLE_NAMES_CONFIG); + options.getOptional(Db2IncrementalSourceOptions.TABLE_NAMES_CONFIG); if (tableConfigs.isPresent()) { catalogTables = CatalogTableUtils.mergeCatalogTableConfig( @@ -127,7 +140,102 @@ public class Db2IncrementalSourceFactory implements TableSourceFactory { tableConfigs.get(), text -> TablePath.of(text, true)); } - return new Db2IncrementalSource(context.getOptions(), catalogTables); + return new Db2IncrementalSource(options, catalogTables); }; } + + /** + * Resolves configured table names against the DB2 catalog before they are used anywhere. + * + * <p>The catalog is opened only when folding would change at least one unquoted name. That + * keeps uppercase and quoted configurations on the previous single-connection path, while + * lowercase unquoted names get one additional short-lived connection to check whether the + * configured spelling exists; it is kept when it does and folded otherwise. Without a DB2 + * catalog factory no configured spelling can be confirmed, so the names are folded + * unconditionally. + */ + static ReadonlyConfig resolveDb2Options(ReadonlyConfig options, ClassLoader classLoader) { + boolean foldingChangesAnyName = + Stream.concat( + options.getOptional(ConnectorCommonOptions.TABLE_NAMES) + .map(List::stream) + .orElseGet(Stream::empty), + options.getOptional(Db2IncrementalSourceOptions.TABLE_NAMES_CONFIG) + .map( + tables -> + tables.stream() + .map( + JdbcSourceTableConfig + ::getTable)) + .orElseGet(Stream::empty)) + .anyMatch( + name -> + !TableDiscoveryUtils.normalizeConfiguredTableName(name) + .equals(name)); + if (!foldingChangesAnyName) { + return normalizeDb2Options(options, tablePath -> true); + } + Optional<Catalog> optionalCatalog = + FactoryUtil.createOptionalCatalog( + DatabaseIdentifier.DB_2, options, classLoader, DatabaseIdentifier.DB_2); + if (!optionalCatalog.isPresent()) { + // No catalog can confirm the configured spelling, so every changed name is folded. + return normalizeDb2Options(options, tablePath -> false); + } + try (Catalog catalog = optionalCatalog.get()) { + catalog.open(); + return normalizeDb2Options(options, catalog::tableExists); + } + } + + /** + * Keeps catalog lookup, per-table overrides and capture filters in the same identifier space. + * Each distinct configured name is resolved once through {@code tableExists}; see {@link + * TableDiscoveryUtils#resolveConfiguredTableName(String, Predicate)} for the precedence. Table + * patterns are regular expressions and must not be case-folded here. + */ + static ReadonlyConfig normalizeDb2Options( + ReadonlyConfig options, Predicate<TablePath> tableExists) { + Map<String, String> resolvedNames = new HashMap<>(); + Function<String, String> resolver = + name -> + resolvedNames.computeIfAbsent( + name, + configuredTable -> + TableDiscoveryUtils.resolveConfiguredTableName( + configuredTable, tableExists)); + Map<String, Object> normalized = new LinkedHashMap<>(options.getSourceMap()); + options.getOptional(ConnectorCommonOptions.TABLE_NAMES) + .ifPresent( + names -> + normalized.put( + ConnectorCommonOptions.TABLE_NAMES.key(), + names.stream().map(resolver).collect(Collectors.toList()))); + options.getOptional(Db2IncrementalSourceOptions.TABLE_NAMES_CONFIG) + .ifPresent( + tables -> + normalized.put( + Db2IncrementalSourceOptions.TABLE_NAMES_CONFIG.key(), + tables.stream() + .map(table -> normalizeTableConfig(table, resolver)) + .collect(Collectors.toList()))); + return ReadonlyConfig.fromMap(normalized); + } + + /** + * Preserves user-configured primary keys and split-column override while resolving the table + * name with the same resolver as {@code table-names}. + */ + private static Map<String, Object> normalizeTableConfig( + JdbcSourceTableConfig config, Function<String, String> resolver) { + Map<String, Object> normalized = new LinkedHashMap<>(); + normalized.put("table", resolver.apply(config.getTable())); + if (config.getPrimaryKeys() != null) { + normalized.put("primaryKeys", new ArrayList<>(config.getPrimaryKeys())); + } + if (config.getSnapshotSplitColumn() != null) { + normalized.put("snapshotSplitColumn", config.getSnapshotSplitColumn()); + } + return normalized; + } } diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/reader/fetch/Db2SourceFetchTaskContext.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/reader/fetch/Db2SourceFetchTaskContext.java index 7ad6c83545..86ddab0d42 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/reader/fetch/Db2SourceFetchTaskContext.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/reader/fetch/Db2SourceFetchTaskContext.java @@ -180,14 +180,9 @@ public class Db2SourceFetchTaskContext extends JdbcSourceFetchTaskContext { @Override public void close() { - try { - this.dataConnection.close(); - if (this.metadataConnection != null) { - this.metadataConnection.close(); - } - } catch (SQLException e) { - log.warn("Failed to close connection", e); - } + // A failure closing one connection must not leak the other connection. + closeConnection(dataConnection); + closeConnection(metadataConnection); } @Override @@ -273,6 +268,17 @@ public class Db2SourceFetchTaskContext extends JdbcSourceFetchTaskContext { return loader.load(offset.getOffset()); } + private void closeConnection(Db2Connection connection) { + if (connection != null) { + try { + connection.close(); + } catch (SQLException e) { + log.warn("Failed to close DB2 CDC connection", e); + } + } + } + + /** Extracts optional source metadata without dereferencing an absent source struct. */ public static class Db2EventMetadataProvider implements EventMetadataProvider { @Override @@ -282,7 +288,7 @@ public class Db2SourceFetchTaskContext extends JdbcSourceFetchTaskContext { return null; } final Struct sourceInfo = value.getStruct(Envelope.FieldName.SOURCE); - if (source == null) { + if (sourceInfo == null) { return null; } final Long timestamp = sourceInfo.getInt64(SourceInfo.TIMESTAMP_KEY); @@ -296,7 +302,7 @@ public class Db2SourceFetchTaskContext extends JdbcSourceFetchTaskContext { return null; } final Struct sourceInfo = value.getStruct(Envelope.FieldName.SOURCE); - if (source == null) { + if (sourceInfo == null) { return null; } return Collect.hashMapOf( @@ -311,7 +317,7 @@ public class Db2SourceFetchTaskContext extends JdbcSourceFetchTaskContext { return null; } final Struct sourceInfo = value.getStruct(Envelope.FieldName.SOURCE); - if (source == null) { + if (sourceInfo == null) { return null; } return sourceInfo.getString(SourceInfo.COMMIT_LSN_KEY); diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2CdcConnection.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2CdcConnection.java new file mode 100644 index 0000000000..eeaf2b1ebc --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2CdcConnection.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils; + +import io.debezium.connector.db2.Db2ChangeTable; +import io.debezium.connector.db2.Db2Connection; +import io.debezium.connector.db2.Lsn; +import io.debezium.jdbc.JdbcConfiguration; +import io.debezium.relational.TableId; + +import java.sql.SQLException; +import java.util.HashSet; +import java.util.Set; + +/** + * Maps active ASN Capture registrations to Debezium's change-table lifecycle. + * + * <p>Debezium 1.9.8 maps the advancing NEW/OLD synchronization points to start/stop LSNs. An active + * registration instead starts at OLD and has no stop LSN. The streaming reader can still assign a + * stop LSN when it discovers a successor capture instance. + */ +class Db2CdcConnection extends Db2Connection { + + private static final String CAPTURE_TABLES_QUERY = + "SELECT r.SOURCE_OWNER, r.SOURCE_TABLE, r.CD_TABLE, r.CD_OLD_SYNCHPOINT, " + + "CAST((t.TBSPACEID * 65536 + t.TABLEID) AS INTEGER) " + + "FROM ASNCDC.IBMSNAP_REGISTER r LEFT JOIN SYSCAT.TABLES t " + + "ON r.SOURCE_OWNER = t.TABSCHEMA AND r.SOURCE_TABLE = t.TABNAME " + + "WHERE r.SOURCE_OWNER <> ''"; + + Db2CdcConnection(JdbcConfiguration configuration) { + super(configuration); + } + + /** + * Reads raw capture names once so Debezium can quote mixed-case names without double quoting. + */ + @Override + public Set<Db2ChangeTable> listOfChangeTables() throws SQLException { + return queryAndMap( + CAPTURE_TABLES_QUERY, + resultSet -> { + Set<Db2ChangeTable> tables = new HashSet<>(); + while (resultSet.next()) { + tables.add( + new Db2ChangeTable( + new TableId( + "", resultSet.getString(1), resultSet.getString(2)), + resultSet.getString(3), + resultSet.getInt(5), + Lsn.valueOf(resultSet.getBytes(4)), + Lsn.NULL)); + } + return tables; + }); + } +} diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2ConnectionUtils.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2ConnectionUtils.java index 69aeeddda2..3918f85021 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2ConnectionUtils.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2ConnectionUtils.java @@ -27,6 +27,6 @@ public class Db2ConnectionUtils { public static Db2Connection createDb2Connection(Configuration dbzConfiguration) { final Db2ConnectorConfig connectorConfig = new Db2ConnectorConfig(dbzConfiguration); - return new Db2Connection(JdbcConfiguration.adapt(connectorConfig.getJdbcConfig())); + return new Db2CdcConnection(JdbcConfiguration.adapt(connectorConfig.getJdbcConfig())); } } diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/TableDiscoveryUtils.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/TableDiscoveryUtils.java index ee4a7e8bd8..3660b079e2 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/TableDiscoveryUtils.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/TableDiscoveryUtils.java @@ -17,6 +17,7 @@ package org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils; +import org.apache.seatunnel.api.table.catalog.TablePath; import org.apache.seatunnel.common.utils.SeaTunnelException; import org.slf4j.Logger; @@ -29,7 +30,9 @@ import io.debezium.relational.TableId; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Set; +import java.util.function.Predicate; import java.util.stream.Collectors; /** Utilities to discovery matched tables. */ @@ -72,7 +75,10 @@ public class TableDiscoveryUtils { .collect(Collectors.toSet()); List<String> missingTables = new ArrayList<>(); for (String configuredTable : configuredTables) { - if (!normalizedCapturedTables.contains(toConfiguredDb2TableId(configuredTable))) { + // Options already contain the resolved physical spelling. Accepting a folded twin + // here could validate capture for a different object than the catalog selected. + TableId configuredTableId = toConfiguredDb2TableId(configuredTable); + if (!normalizedCapturedTables.contains(configuredTableId)) { missingTables.add(configuredTable); } } @@ -102,6 +108,44 @@ public class TableDiscoveryUtils { configuredTable.substring(secondDot + 1)); } + /** + * Folds unquoted schema/table names to uppercase, the spelling DB2 gives unquoted identifiers. + * The database is part of the SeaTunnel table identity and retains its configured spelling. + * Double quotes are not interpreted by the catalog lookup or the capture filter, so a name that + * contains a quote character is returned unchanged rather than partially folded. + */ + public static String normalizeConfiguredTableName(String configuredTable) { + if (configuredTable.contains("\"")) { + return configuredTable; + } + int databaseEnd = configuredTable.indexOf('.') + 1; + return configuredTable.substring(0, databaseEnd) + + configuredTable.substring(databaseEnd).toUpperCase(Locale.ROOT); + } + + /** + * Resolves the spelling that catalog lookup, capture filtering and SeaTunnel table identity + * share for one configured table. + * + * <p>Releases up to 3.0.0 passed the configured text through unchanged. The configured spelling + * therefore wins whenever an object with exactly that spelling exists, so an upgraded or + * restored job keeps selecting the same table even when an object with the folded spelling + * exists as well. Only when the configured spelling does not exist is the DB2 folded + * (uppercase) spelling used, which lets an unquoted lowercase name select an ordinary uppercase + * object; if neither exists the folded name is returned and the existing lookups fail fast with + * it. Names that folding leaves unchanged, including names containing quotes, never consult + * {@code tableExists}. + */ + public static String resolveConfiguredTableName( + String configuredTable, Predicate<TablePath> tableExists) { + String foldedTable = normalizeConfiguredTableName(configuredTable); + if (foldedTable.equals(configuredTable) + || tableExists.test(TablePath.of(configuredTable))) { + return configuredTable; + } + return foldedTable; + } + /** * Debezium change-table discovery emits empty-catalog {@link TableId}s for Db2, so startup * validation normalizes both sides to the same shape before comparing them. diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2DialectTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2DialectTest.java new file mode 100644 index 0000000000..e2648017eb --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2DialectTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.cdc.db2.source; + +import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.catalog.ConstraintKey; +import org.apache.seatunnel.api.table.catalog.PhysicalColumn; +import org.apache.seatunnel.api.table.catalog.PrimaryKey; +import org.apache.seatunnel.api.table.catalog.TableIdentifier; +import org.apache.seatunnel.api.table.catalog.TableSchema; +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.connectors.seatunnel.cdc.db2.config.Db2SourceConfig; +import org.apache.seatunnel.connectors.seatunnel.cdc.db2.config.Db2SourceConfigFactory; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import io.debezium.relational.TableId; + +import java.util.Collections; +import java.util.HashMap; + +/** Protects catalog-free DB2 key lookup and absent catalog entries during split planning. */ +class Db2DialectTest { + + @Test + void shouldReturnEmptyKeysForMissingCatalogTable() { + Db2Dialect dialect = dialect(null); + TableId table = new TableId("", "DB2INST1", "UNKNOWN"); + + Assertions.assertEquals(Collections.emptyList(), dialect.getConstraintKeys(null, table)); + Assertions.assertFalse(dialect.getPrimaryKey(null, table).isPresent()); + } + + @Test + void shouldPreserveConfiguredKeysWithoutDatabaseSegment() { + PrimaryKey primaryKey = PrimaryKey.of("PK", Collections.singletonList("ID")); + ConstraintKey uniqueKey = + ConstraintKey.of( + ConstraintKey.ConstraintType.UNIQUE_KEY, + "UK", + Collections.singletonList( + ConstraintKey.ConstraintKeyColumn.of( + "ID", ConstraintKey.ColumnSortType.ASC))); + CatalogTable table = + CatalogTable.of( + TableIdentifier.of("default", "E2E", "DB2INST1", "CUSTOMERS"), + TableSchema.builder() + .column( + PhysicalColumn.of( + "ID", BasicType.INT_TYPE, 10L, false, null, null)) + .primaryKey(primaryKey) + .constraintKey(Collections.singletonList(uniqueKey)) + .build(), + new HashMap<>(), + Collections.emptyList(), + null); + Db2Dialect dialect = dialect(table); + TableId runtimeId = new TableId("", "DB2INST1", "CUSTOMERS"); + + Assertions.assertEquals( + Collections.singletonList("ID"), + dialect.getPrimaryKey(null, runtimeId).get().getColumnNames()); + Assertions.assertEquals( + Collections.singletonList(uniqueKey), dialect.getConstraintKeys(null, runtimeId)); + } + + private static Db2Dialect dialect(CatalogTable table) { + Db2SourceConfigFactory factory = Mockito.mock(Db2SourceConfigFactory.class); + Mockito.when(factory.create(0)).thenReturn(Mockito.mock(Db2SourceConfig.class)); + return new Db2Dialect( + factory, + table == null ? Collections.emptyList() : Collections.singletonList(table)); + } +} diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2IncrementalSourceFactoryTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2IncrementalSourceFactoryTest.java index 721cf79aad..891a26dd2a 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2IncrementalSourceFactoryTest.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2IncrementalSourceFactoryTest.java @@ -21,7 +21,9 @@ import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.configuration.util.ConfigValidator; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.configuration.util.OptionValidationException; +import org.apache.seatunnel.api.options.ConnectorCommonOptions; import org.apache.seatunnel.api.table.catalog.TablePath; +import org.apache.seatunnel.connectors.cdc.base.config.JdbcSourceTableConfig; import org.apache.seatunnel.connectors.seatunnel.cdc.db2.config.Db2SourceConfig; import org.apache.seatunnel.connectors.seatunnel.cdc.db2.config.Db2SourceConfigFactory; @@ -34,11 +36,16 @@ import org.mockito.Mockito; import io.debezium.relational.TableId; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.stream.Stream; +/** + * Covers DB2 factory contracts and consistent unquoted table names without losing split overrides. + */ class Db2IncrementalSourceFactoryTest { private static final String VALID_URL = "jdbc:db2://localhost:50000/seatunnel"; @@ -149,4 +156,151 @@ class Db2IncrementalSourceFactoryTest { new TableId("", "DB2INST1", "CUSTOMERS"), dialect.toTableId(TablePath.of("SAMPLE", "DB2INST1", "CUSTOMERS"))); } + + @Test + void shouldNormalizeSchemaAndTableWithoutChangingDatabaseOrSplitOverrides() { + Map<String, Object> tableConfig = new HashMap<>(); + tableConfig.put("table", "e2e.db2inst1.customers"); + tableConfig.put("primaryKeys", Collections.singletonList("ID")); + tableConfig.put("snapshotSplitColumn", "ID"); + Map<String, Object> raw = new HashMap<>(); + raw.put("table-names", Collections.singletonList("e2e.db2inst1.customers")); + raw.put("table-names-config", Collections.singletonList(tableConfig)); + raw.put("database-names", Collections.singletonList("e2e")); + raw.put("base-url", "jdbc:db2://localhost:50000/e2e"); + raw.put("exactly_once", true); + raw.put("enable_concurrent_read", false); + ReadonlyConfig original = ReadonlyConfig.fromMap(raw); + + ReadonlyConfig normalized = + Db2IncrementalSourceFactory.normalizeDb2Options( + original, Db2IncrementalSourceFactoryTest::isFoldedPath); + JdbcSourceTableConfig table = + normalized.get(Db2IncrementalSourceOptions.TABLE_NAMES_CONFIG).get(0); + + Assertions.assertEquals( + Collections.singletonList("e2e.DB2INST1.CUSTOMERS"), + normalized.get(ConnectorCommonOptions.TABLE_NAMES)); + Assertions.assertEquals("e2e.DB2INST1.CUSTOMERS", table.getTable()); + TablePath tablePath = TablePath.of(table.getTable(), true); + Assertions.assertEquals("e2e", tablePath.getDatabaseName()); + Assertions.assertEquals("DB2INST1", tablePath.getSchemaName()); + Assertions.assertEquals("CUSTOMERS", tablePath.getTableName()); + Assertions.assertEquals( + Collections.singletonList("e2e"), + normalized.get(Db2IncrementalSourceOptions.DATABASE_NAMES)); + Assertions.assertEquals(Collections.singletonList("ID"), table.getPrimaryKeys()); + Assertions.assertEquals("ID", table.getSnapshotSplitColumn()); + Assertions.assertEquals( + "jdbc:db2://localhost:50000/e2e", normalized.get(Db2IncrementalSourceOptions.URL)); + Assertions.assertTrue(normalized.get(Db2IncrementalSourceOptions.EXACTLY_ONCE)); + Assertions.assertEquals( + Collections.singletonList("e2e.db2inst1.customers"), + original.get(ConnectorCommonOptions.TABLE_NAMES)); + Assertions.assertEquals("e2e.db2inst1.customers", tableConfig.get("table")); + + // When both spellings exist the configured object keeps its identity in both options. + ReadonlyConfig bothExist = + Db2IncrementalSourceFactory.normalizeDb2Options(original, candidate -> true); + Assertions.assertEquals( + Collections.singletonList("e2e.db2inst1.customers"), + bothExist.get(ConnectorCommonOptions.TABLE_NAMES)); + Assertions.assertEquals( + "e2e.db2inst1.customers", + bothExist.get(Db2IncrementalSourceOptions.TABLE_NAMES_CONFIG).get(0).getTable()); + } + + @Test + void shouldKeepTablePatternAndQuotedSpellingUnchanged() { + Map<String, Object> raw = new HashMap<>(); + raw.put("table-pattern", "e2e.DB2INST1.Customer.*"); + ReadonlyConfig normalized = + Db2IncrementalSourceFactory.normalizeDb2Options( + ReadonlyConfig.fromMap(raw), tablePath -> true); + Assertions.assertEquals( + "e2e.DB2INST1.Customer.*", normalized.get(ConnectorCommonOptions.TABLE_PATTERN)); + Assertions.assertFalse( + normalized.getOptional(ConnectorCommonOptions.TABLE_NAMES).isPresent()); + + raw.remove("table-pattern"); + raw.put("table-names", Collections.singletonList("E2E.DB2INST1.\"Customers\"")); + normalized = + Db2IncrementalSourceFactory.normalizeDb2Options( + ReadonlyConfig.fromMap(raw), tablePath -> true); + Assertions.assertEquals( + Collections.singletonList("E2E.DB2INST1.\"Customers\""), + normalized.get(ConnectorCommonOptions.TABLE_NAMES)); + } + + /** + * Objects created with quoted lowercase identifiers were matched by unquoted lowercase + * configuration before folding existed. When only the configured spelling exists, both {@code + * table-names} and {@code table-names-config.table} must keep it; otherwise the upgraded job + * fails at startup with a table that is no longer found or captured. + */ + @Test + void shouldKeepConfiguredSpellingWhenOnlyLowercaseCreatedObjectExists() { + Map<String, Object> tableConfig = new HashMap<>(); + tableConfig.put("table", "e2e.db2inst1.customers"); + tableConfig.put("primaryKeys", Collections.singletonList("id")); + Map<String, Object> raw = new HashMap<>(); + raw.put("table-names", Arrays.asList("e2e.db2inst1.customers", "e2e.db2inst1.orders")); + raw.put("table-names-config", Collections.singletonList(tableConfig)); + TablePath lowercaseCustomers = TablePath.of("e2e", "db2inst1", "customers"); + TablePath uppercaseOrders = TablePath.of("e2e", "DB2INST1", "ORDERS"); + + ReadonlyConfig normalized = + Db2IncrementalSourceFactory.normalizeDb2Options( + ReadonlyConfig.fromMap(raw), + tablePath -> + tablePath.equals(lowercaseCustomers) + || tablePath.equals(uppercaseOrders)); + + Assertions.assertEquals( + Arrays.asList("e2e.db2inst1.customers", "e2e.DB2INST1.ORDERS"), + normalized.get(ConnectorCommonOptions.TABLE_NAMES)); + JdbcSourceTableConfig table = + normalized.get(Db2IncrementalSourceOptions.TABLE_NAMES_CONFIG).get(0); + Assertions.assertEquals("e2e.db2inst1.customers", table.getTable()); + Assertions.assertEquals(Collections.singletonList("id"), table.getPrimaryKeys()); + } + + @Test + void shouldUseNormalizedNamesForDebeziumCaptureFilter() { + Map<String, Object> raw = new HashMap<>(); + raw.put("table-names", Collections.singletonList("e2e.db2inst1.customers")); + raw.put("username", "user"); + raw.put("password", "password"); + raw.put("exactly_once", true); + raw.put("enable_concurrent_read", false); + Db2SourceConfigFactory factory = new Db2SourceConfigFactory(); + factory.fromReadonlyConfig( + Db2IncrementalSourceFactory.normalizeDb2Options( + ReadonlyConfig.fromMap(raw), + Db2IncrementalSourceFactoryTest::isFoldedPath)); + factory.hostname("localhost").port(50000).databaseList("e2e"); + Db2SourceConfig config = factory.create(0); + + Assertions.assertTrue( + config.getTableFilters() + .dataCollectionFilter() + .isIncluded(new TableId("", "DB2INST1", "CUSTOMERS"))); + Assertions.assertFalse( + config.getTableFilters() + .dataCollectionFilter() + .isIncluded(new TableId("", "DB2INST1", "ORDERS"))); + Assertions.assertTrue(config.isExactlyOnce()); + Assertions.assertFalse(config.isEnableConcurrentRead()); + } + + /** + * Simulates a DB2 catalog that only contains ordinary objects, whose schema and table names DB2 + * stores in the folded uppercase spelling. + */ + private static boolean isFoldedPath(TablePath tablePath) { + return tablePath.getSchemaName().equals(tablePath.getSchemaName().toUpperCase(Locale.ROOT)) + && tablePath + .getTableName() + .equals(tablePath.getTableName().toUpperCase(Locale.ROOT)); + } } diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2SavepointHistoryRestoreTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2SavepointHistoryRestoreTest.java new file mode 100644 index 0000000000..5d7035f746 --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2SavepointHistoryRestoreTest.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.cdc.db2.source; + +import org.apache.seatunnel.connectors.cdc.base.dialect.JdbcDataSourceDialect; +import org.apache.seatunnel.connectors.cdc.base.source.split.IncrementalSplit; +import org.apache.seatunnel.connectors.cdc.debezium.ConnectTableChangeSerializer; +import org.apache.seatunnel.connectors.cdc.debezium.EmbeddedDatabaseHistory; +import org.apache.seatunnel.connectors.seatunnel.cdc.db2.config.Db2SourceConfig; +import org.apache.seatunnel.connectors.seatunnel.cdc.db2.config.Db2SourceConfigFactory; +import org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.offset.LsnOffset; +import org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.reader.fetch.Db2SourceFetchTaskContext; +import org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils.Db2ConnectionUtils; + +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.json.JsonConverter; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import io.debezium.config.Configuration; +import io.debezium.connector.db2.Db2Connection; +import io.debezium.relational.Column; +import io.debezium.relational.Table; +import io.debezium.relational.TableId; +import io.debezium.relational.history.TableChanges; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.sql.Types; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Restores serialized split history through the production JDBC registration path without live + * schema fallback, including the catalog-first contract used by MySQL checkpoints. + */ +class Db2SavepointHistoryRestoreTest { + + @Test + void shouldRestoreDb2SchemaFirstHistoryWithoutLiveSchemaFallback() throws Exception { + assertRestored( + Arrays.asList( + new TableId("", "DB2INST1", "CUSTOMERS"), + new TableId(null, "部门.一", "表.二"))); + } + + @Test + void shouldPreserveCatalogFirstAndThreePartCheckpointIds() throws Exception { + assertRestored( + Arrays.asList( + new TableId("inventory", null, "customers"), + new TableId("database", "schema", "table"))); + } + + private static void assertRestored(List<TableId> tableIds) throws Exception { + Map<TableId, byte[]> history = new HashMap<>(); + JsonConverter converter = new JsonConverter(); + converter.configure(Collections.singletonMap("schemas.enable", true), false); + try { + for (TableId tableId : tableIds) { + Table table = + Table.editor() + .tableId(tableId) + .addColumn( + Column.editor() + .name("ID") + .jdbcType(Types.INTEGER) + .type("INTEGER") + .position(1) + .optional(false) + .create()) + .setPrimaryKeyNames(Collections.singletonList("ID")) + .create(); + Struct change = + new ConnectTableChangeSerializer() + .serialize(new TableChanges().create(table)) + .get(0); + history.put(tableId, converter.fromConnectData("history", change.schema(), change)); + } + } finally { + converter.close(); + } + IncrementalSplit split = + new IncrementalSplit( + "stream", + tableIds, + LsnOffset.INITIAL_OFFSET, + LsnOffset.NO_STOPPING_OFFSET, + Collections.emptyList(), + Collections.emptyList(), + history); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ObjectOutputStream output = new ObjectOutputStream(bytes)) { + output.writeObject(split); + } + try (ObjectInputStream input = + new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + split = (IncrementalSplit) input.readObject(); + } + + Db2SourceConfigFactory factory = new Db2SourceConfigFactory(); + factory.hostname("localhost") + .port(50000) + .username("user") + .password("password") + .databaseList("E2E"); + Db2SourceConfig config = factory.create(0); + JdbcDataSourceDialect dialect = Mockito.mock(JdbcDataSourceDialect.class); + Mockito.when(dialect.queryTableSchema(Mockito.any(), Mockito.any())) + .thenThrow(new AssertionError("Saved history must not fall back to live metadata")); + Db2Connection connection = Mockito.mock(Db2Connection.class); + String historyName = + config.getDbzConfiguration() + .getString(EmbeddedDatabaseHistory.DATABASE_HISTORY_INSTANCE_NAME); + try (MockedStatic<Db2ConnectionUtils> connections = + Mockito.mockStatic(Db2ConnectionUtils.class)) { + connections + .when( + () -> + Db2ConnectionUtils.createDb2Connection( + Mockito.any(Configuration.class))) + .thenReturn(connection); + HistoryContext context = new HistoryContext(config, dialect); + try { + context.register(split); + Collection<TableChanges.TableChange> restored = + EmbeddedDatabaseHistory.removeHistory(historyName); + List<Table> tables = + restored.stream() + .map(TableChanges.TableChange::getTable) + .collect(Collectors.toList()); + Assertions.assertEquals(tableIds.size(), tables.size()); + Assertions.assertEquals( + tableIds, tables.stream().map(Table::id).collect(Collectors.toList())); + for (int i = 0; i < tables.size(); i++) { + String catalog = tableIds.get(i).catalog(); + Assertions.assertEquals( + catalog == null || catalog.isEmpty() ? null : catalog, + tables.get(i).id().catalog()); + Assertions.assertEquals(tableIds.get(i).schema(), tables.get(i).id().schema()); + Assertions.assertEquals(tableIds.get(i).table(), tables.get(i).id().table()); + Assertions.assertEquals( + Collections.singletonList("ID"), tables.get(i).primaryKeyColumnNames()); + Assertions.assertEquals( + Types.INTEGER, tables.get(i).columnWithName("ID").jdbcType()); + Assertions.assertFalse(tables.get(i).columnWithName("ID").isOptional()); + } + } finally { + context.close(); + EmbeddedDatabaseHistory.removeHistory(historyName); + } + } + } + + /** + * Exposes the protected production history entry point without changing runtime code for tests. + */ + private static class HistoryContext extends Db2SourceFetchTaskContext { + HistoryContext(Db2SourceConfig config, JdbcDataSourceDialect dialect) { + super(config, dialect); + } + + void register(IncrementalSplit split) { + registerDatabaseHistory(split, getDataConnection()); + } + } +} diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2SourceFetchTaskContextTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2SourceFetchTaskContextTest.java new file mode 100644 index 0000000000..0dc04586ed --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2SourceFetchTaskContextTest.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.cdc.db2.source; + +import org.apache.seatunnel.connectors.cdc.base.dialect.JdbcDataSourceDialect; +import org.apache.seatunnel.connectors.cdc.base.source.split.IncrementalSplit; +import org.apache.seatunnel.connectors.cdc.debezium.EmbeddedDatabaseHistory; +import org.apache.seatunnel.connectors.seatunnel.cdc.db2.config.Db2SourceConfig; +import org.apache.seatunnel.connectors.seatunnel.cdc.db2.config.Db2SourceConfigFactory; +import org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.offset.LsnOffset; +import org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.reader.fetch.Db2SourceFetchTaskContext; +import org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils.Db2ConnectionUtils; +import org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils.Db2Utils; + +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.SchemaBuilder; +import org.apache.kafka.connect.data.Struct; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import io.debezium.config.Configuration; +import io.debezium.connector.db2.Db2Connection; +import io.debezium.connector.db2.Db2ConnectorConfig; +import io.debezium.connector.db2.Db2DatabaseSchema; +import io.debezium.connector.db2.SourceInfo; +import io.debezium.data.Envelope; +import io.debezium.relational.TableId; + +import java.sql.SQLException; +import java.time.Instant; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Covers optional event metadata and independent connection cleanup through the real fetch context. + */ +class Db2SourceFetchTaskContextTest { + + @Test + void shouldIgnoreAbsentSourceStructWithNonNullTableId() { + Struct value = envelope(null); + TableId table = new TableId("", "DB2INST1", "CUSTOMERS"); + Db2SourceFetchTaskContext.Db2EventMetadataProvider provider = provider(); + + Assertions.assertNull(provider.getEventTimestamp(table, null, null, value)); + Assertions.assertNull(provider.getEventSourcePosition(table, null, null, value)); + Assertions.assertNull(provider.getTransactionId(table, null, null, value)); + Assertions.assertNull(provider.getEventTimestamp(table, null, null, null)); + Assertions.assertNull(provider.getEventSourcePosition(table, null, null, null)); + Assertions.assertNull(provider.getTransactionId(table, null, null, null)); + } + + @Test + void shouldReadSourceMetadataIndependentlyOfCollectionId() { + Struct source = + new Struct(sourceSchema()) + .put(SourceInfo.TIMESTAMP_KEY, 1234L) + .put(SourceInfo.COMMIT_LSN_KEY, "commit") + .put(SourceInfo.CHANGE_LSN_KEY, "change"); + Struct value = envelope(source); + Db2SourceFetchTaskContext.Db2EventMetadataProvider provider = provider(); + Map<String, String> expected = new HashMap<>(); + expected.put(SourceInfo.COMMIT_LSN_KEY, "commit"); + expected.put(SourceInfo.CHANGE_LSN_KEY, "change"); + + Assertions.assertEquals( + Instant.ofEpochMilli(1234L), provider.getEventTimestamp(null, null, null, value)); + Assertions.assertEquals(expected, provider.getEventSourcePosition(null, null, null, value)); + Assertions.assertEquals("commit", provider.getTransactionId(null, null, null, value)); + source.put(SourceInfo.TIMESTAMP_KEY, null); + Assertions.assertNull(provider.getEventTimestamp(null, null, null, value)); + } + + @Test + void shouldReleaseMetadataConnectionWhenDataConnectionCloseFails() throws Exception { + Db2Connection data = Mockito.mock(Db2Connection.class); + Db2Connection metadata = Mockito.mock(Db2Connection.class); + AtomicBoolean metadataClosed = new AtomicBoolean(); + Mockito.doThrow(new SQLException("data close failure")).when(data).close(); + Mockito.doAnswer( + invocation -> { + metadataClosed.set(true); + return null; + }) + .when(metadata) + .close(); + + Db2SourceFetchTaskContext context = configureContext(data, metadata); + Assertions.assertSame(metadata, context.getMetadataConnection()); + context.close(); + + Assertions.assertTrue( + metadataClosed.get(), + "The auxiliary DB2 connection must be released after primary close fails"); + } + + private static Db2SourceFetchTaskContext configureContext( + Db2Connection data, Db2Connection metadata) { + Db2SourceConfigFactory factory = new Db2SourceConfigFactory(); + factory.hostname("localhost") + .port(50000) + .username("user") + .password("password") + .databaseList("E2E"); + Db2SourceConfig config = factory.create(0); + JdbcDataSourceDialect dialect = Mockito.mock(JdbcDataSourceDialect.class); + Db2DatabaseSchema schema = Mockito.mock(Db2DatabaseSchema.class); + try (MockedStatic<Db2ConnectionUtils> connections = + Mockito.mockStatic(Db2ConnectionUtils.class); + MockedStatic<Db2Utils> schemas = Mockito.mockStatic(Db2Utils.class)) { + connections + .when( + () -> + Db2ConnectionUtils.createDb2Connection( + Mockito.any(Configuration.class))) + .thenReturn(data, metadata); + schemas.when( + () -> + Db2Utils.createDb2DatabaseSchema( + Mockito.any(Db2ConnectorConfig.class), + Mockito.same(data))) + .thenReturn(schema); + Db2SourceFetchTaskContext context = new Db2SourceFetchTaskContext(config, dialect); + context.configure( + new IncrementalSplit( + "stream", + Collections.emptyList(), + LsnOffset.INITIAL_OFFSET, + LsnOffset.NO_STOPPING_OFFSET, + Collections.emptyList())); + EmbeddedDatabaseHistory.removeHistory( + config.getDbzConfiguration() + .getString(EmbeddedDatabaseHistory.DATABASE_HISTORY_INSTANCE_NAME)); + return context; + } + } + + private static Db2SourceFetchTaskContext.Db2EventMetadataProvider provider() { + return new Db2SourceFetchTaskContext.Db2EventMetadataProvider(); + } + + private static Struct envelope(Struct source) { + return new Struct( + SchemaBuilder.struct() + .field(Envelope.FieldName.SOURCE, sourceSchema()) + .build()) + .put(Envelope.FieldName.SOURCE, source); + } + + private static Schema sourceSchema() { + return SchemaBuilder.struct() + .optional() + .field(SourceInfo.TIMESTAMP_KEY, Schema.OPTIONAL_INT64_SCHEMA) + .field(SourceInfo.COMMIT_LSN_KEY, Schema.OPTIONAL_STRING_SCHEMA) + .field(SourceInfo.CHANGE_LSN_KEY, Schema.OPTIONAL_STRING_SCHEMA) + .build(); + } +} diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2CdcConnectionTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2CdcConnectionTest.java new file mode 100644 index 0000000000..d0707faccc --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2CdcConnectionTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import io.debezium.connector.db2.Db2ChangeTable; +import io.debezium.connector.db2.Lsn; +import io.debezium.relational.TableId; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Set; + +/** + * Protects active Capture watermarks and raw capture-name quoting at the JDBC discovery boundary. + */ +class Db2CdcConnectionTest { + + @Test + void shouldKeepActiveCaptureOpenAfterOldSynchpoint() throws SQLException { + Db2ChangeTable table = discoverCapture("DB2INST1_CUSTOMERS", lsn(7)); + + Assertions.assertEquals(new TableId("", "DB2INST1", "CUSTOMERS"), table.getSourceTableId()); + Assertions.assertEquals(42, table.getChangeTableObjectId()); + Assertions.assertEquals(Lsn.valueOf(lsn(7)), table.getStartLsn()); + Assertions.assertEquals(Lsn.NULL, table.getStopLsn()); + Assertions.assertFalse(table.getStopLsn().isAvailable()); + Assertions.assertEquals( + new TableId("", "ASNCDC", "DB2INST1_CUSTOMERS"), table.getChangeTableId()); + + // The reader remains free to close an older instance at a successor's start LSN. + table.setStopLsn(Lsn.valueOf(lsn(19))); + Assertions.assertEquals(Lsn.valueOf(lsn(19)), table.getStopLsn()); + } + + @Test + void shouldQuoteMixedCaseCaptureNameExactlyOnce() throws SQLException { + Db2ChangeTable table = discoverCapture("CustomersCapture", lsn(3)); + + Assertions.assertEquals("\"CustomersCapture\"", table.getCaptureInstance()); + Assertions.assertEquals( + new TableId("", "ASNCDC", "\"CustomersCapture\""), table.getChangeTableId()); + Assertions.assertEquals(Lsn.valueOf(lsn(3)), table.getStartLsn()); + Assertions.assertEquals(Lsn.NULL, table.getStopLsn()); + } + + @Test + void shouldPreserveUnavailableOldSynchpoint() throws SQLException { + Db2ChangeTable table = discoverCapture("DB2INST1_CUSTOMERS", null); + + Assertions.assertEquals(Lsn.NULL, table.getStartLsn()); + Assertions.assertEquals(Lsn.NULL, table.getStopLsn()); + } + + private static Db2ChangeTable discoverCapture(String captureName, byte[] oldSynchpoint) + throws SQLException { + Db2CdcConnection connection = + Mockito.mock(Db2CdcConnection.class, Mockito.CALLS_REAL_METHODS); + Connection jdbc = Mockito.mock(Connection.class); + Statement statement = Mockito.mock(Statement.class); + ResultSet rows = Mockito.mock(ResultSet.class); + Mockito.doReturn(jdbc).when(connection).connection(); + Mockito.when(jdbc.createStatement()).thenReturn(statement); + // The returned column is OLD, never the advancing NEW synchronization point. + Mockito.when(statement.executeQuery(Mockito.contains("r.CD_OLD_SYNCHPOINT"))) + .thenReturn(rows); + Mockito.when(rows.next()).thenReturn(true, false); + Mockito.when(rows.getString(1)).thenReturn("DB2INST1"); + Mockito.when(rows.getString(2)).thenReturn("CUSTOMERS"); + Mockito.when(rows.getString(3)).thenReturn(captureName); + Mockito.when(rows.getBytes(4)).thenReturn(oldSynchpoint); + Mockito.when(rows.getInt(5)).thenReturn(42); + + Set<Db2ChangeTable> tables = connection.listOfChangeTables(); + Assertions.assertEquals(1, tables.size()); + return tables.iterator().next(); + } + + private static byte[] lsn(int value) { + byte[] bytes = new byte[16]; + bytes[15] = (byte) value; + return bytes; + } +} diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/TableDiscoveryUtilsTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/TableDiscoveryUtilsTest.java index c41634bac6..bbb0b743e4 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/TableDiscoveryUtilsTest.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/TableDiscoveryUtilsTest.java @@ -17,6 +17,7 @@ package org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils; +import org.apache.seatunnel.api.table.catalog.TablePath; import org.apache.seatunnel.common.utils.SeaTunnelException; import org.junit.jupiter.api.Assertions; @@ -66,4 +67,101 @@ class TableDiscoveryUtilsTest { Collections.singletonList( new TableId("", "DB2INST1", "CUSTOMERS")))); } + + @Test + void shouldFoldConfiguredUnquotedNamesAndKeepCapturedCase() { + Assertions.assertEquals( + "e2e.DB2INST1.CUSTOMERS", + TableDiscoveryUtils.normalizeConfiguredTableName("e2e.db2inst1.customers")); + Assertions.assertEquals( + "testdb.DB2INST1.CUSTOMERS", + TableDiscoveryUtils.normalizeConfiguredTableName("testdb.DB2INST1.CUSTOMERS")); + Assertions.assertEquals( + new TableId("", "DB2INST1", "CUSTOMERS"), + TableDiscoveryUtils.toConfiguredDb2TableId( + TableDiscoveryUtils.normalizeConfiguredTableName( + "e2e.db2inst1.customers"))); + Assertions.assertDoesNotThrow( + () -> + TableDiscoveryUtils.validateExplicitCaptureTables( + Collections.singletonList( + TableDiscoveryUtils.resolveConfiguredTableName( + "e2e.db2inst1.customers", + TablePath.of("e2e", "DB2INST1", "CUSTOMERS") + ::equals)), + Collections.singletonList( + new TableId("", "DB2INST1", "CUSTOMERS")))); + Assertions.assertEquals( + new TableId("", "MixedSchema", "Customers"), + TableDiscoveryUtils.normalizeCapturedTableId( + new TableId("E2E", "MixedSchema", "Customers"))); + SeaTunnelException error = + Assertions.assertThrows( + SeaTunnelException.class, + () -> + TableDiscoveryUtils.validateExplicitCaptureTables( + Collections.singletonList("e2e.db2inst1.orders"), + Collections.singletonList( + new TableId("", "DB2INST1", "CUSTOMERS")))); + Assertions.assertTrue(error.getMessage().contains("e2e.db2inst1.orders")); + } + + /** + * A table created with quoted lowercase identifiers and configured unquoted in lowercase was + * captured before folding existed. The resolver must keep that spelling whenever the lowercase + * object exists, including when an uppercase object exists as well, fold only when the + * configured spelling does not exist, and startup validation must accept the lowercase capture + * metadata for the lowercase configuration. + */ + @Test + void shouldKeepCapturingLowercaseCreatedObjectConfiguredUnquoted() { + TablePath lowercaseCustomers = TablePath.of("e2e", "db2inst1", "customers"); + TablePath uppercaseCustomers = TablePath.of("e2e", "DB2INST1", "CUSTOMERS"); + + Assertions.assertEquals( + "e2e.db2inst1.customers", + TableDiscoveryUtils.resolveConfiguredTableName( + "e2e.db2inst1.customers", lowercaseCustomers::equals)); + // Both spellings exist: the configured object selected by releases up to 3.0.0 must win. + Assertions.assertEquals( + "e2e.db2inst1.customers", + TableDiscoveryUtils.resolveConfiguredTableName( + "e2e.db2inst1.customers", + tablePath -> + tablePath.equals(lowercaseCustomers) + || tablePath.equals(uppercaseCustomers))); + SeaTunnelException captureMismatch = + Assertions.assertThrows( + SeaTunnelException.class, + () -> + TableDiscoveryUtils.validateExplicitCaptureTables( + Collections.singletonList( + TableDiscoveryUtils.resolveConfiguredTableName( + "e2e.db2inst1.customers", table -> true)), + Collections.singletonList( + new TableId("", "DB2INST1", "CUSTOMERS")))); + Assertions.assertTrue(captureMismatch.getMessage().contains("e2e.db2inst1.customers")); + Assertions.assertEquals( + "e2e.DB2INST1.CUSTOMERS", + TableDiscoveryUtils.resolveConfiguredTableName( + "e2e.db2inst1.customers", uppercaseCustomers::equals)); + Assertions.assertEquals( + "e2e.DB2INST1.CUSTOMERS", + TableDiscoveryUtils.resolveConfiguredTableName( + "e2e.db2inst1.customers", tablePath -> false)); + Assertions.assertEquals( + "e2e.db2inst1.\"customers\"", + TableDiscoveryUtils.resolveConfiguredTableName( + "e2e.db2inst1.\"customers\"", + tablePath -> { + throw new AssertionError("quoted names must not query the catalog"); + })); + + Assertions.assertDoesNotThrow( + () -> + TableDiscoveryUtils.validateExplicitCaptureTables( + Collections.singletonList("e2e.db2inst1.customers"), + Collections.singletonList( + new TableId("", "db2inst1", "customers")))); + } } diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/split/VitessTableSchemaStateTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/split/VitessTableSchemaStateTest.java new file mode 100644 index 0000000000..881570e0f5 --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/split/VitessTableSchemaStateTest.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source.split; + +import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.catalog.PhysicalColumn; +import org.apache.seatunnel.api.table.catalog.PrimaryKey; +import org.apache.seatunnel.api.table.catalog.TableIdentifier; +import org.apache.seatunnel.api.table.catalog.TableSchema; +import org.apache.seatunnel.api.table.type.BasicType; + +import org.apache.kafka.connect.data.SchemaAndValue; +import org.apache.kafka.connect.json.JsonConverter; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import io.debezium.connector.vitess.VitessDatabaseSchema; +import io.debezium.relational.Column; +import io.debezium.relational.Table; +import io.debezium.relational.TableId; + +import java.nio.charset.StandardCharsets; +import java.sql.Types; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** Protects the schema-first table identity required by Vitess FIELD and restored ROW lookup. */ +class VitessTableSchemaStateTest { + + @Test + void shouldRestoreFieldSchemaWithTheDecoderTableIdentity() { + Table table = + Table.editor() + .tableId(new TableId(null, "commerce", "orders")) + .addColumn( + Column.editor() + .name("ID") + .jdbcType(Types.INTEGER) + .type("INTEGER") + .position(1) + .optional(false) + .create()) + .setPrimaryKeyNames(Collections.singletonList("ID")) + .create(); + + Table restored = + VitessTableSchemaState.deserialize(VitessTableSchemaState.serialize(table)); + + assertSchemaFirstIdentity(restored); + Assertions.assertEquals(Collections.singletonList("ID"), restored.primaryKeyColumnNames()); + Assertions.assertEquals(Types.INTEGER, restored.columnWithName("ID").jdbcType()); + Assertions.assertFalse(restored.columnWithName("ID").isOptional()); + } + + @Test + void shouldKeepCatalogBootstrapAndFieldHistoryInTheSameNamespace() { + CatalogTable catalog = + CatalogTable.of( + TableIdentifier.of("default", "commerce", null, "orders"), + TableSchema.builder() + .column( + PhysicalColumn.of( + "ID", BasicType.INT_TYPE, 10L, false, null, null)) + .primaryKey(PrimaryKey.of("PK", Collections.singletonList("ID"))) + .build(), + new HashMap<>(), + Collections.emptyList(), + null); + Map<String, byte[]> schemas = + VitessTableSchemaState.serializeCatalogTables(Collections.singletonList(catalog)); + Assertions.assertEquals(Collections.singleton("\"commerce\".\"orders\""), schemas.keySet()); + + Table restored = VitessTableSchemaState.deserialize(schemas.values().iterator().next()); + + assertSchemaFirstIdentity(restored); + Assertions.assertEquals(Collections.singletonList("ID"), restored.primaryKeyColumnNames()); + Assertions.assertEquals(Types.INTEGER, restored.columnWithName("ID").jdbcType()); + } + + @Test + void shouldReadPreFixJsonLayoutWithTheDecoderTableIdentity() { + // Fixed layout from the pre-PR CHANGE_SCHEMA and writer; no current serializer builds it. + byte[] checkpoint = + ("{\"schema\":{\"type\":\"struct\",\"fields\":[{\"type\":\"string\",\"optional\":false,\"field\":\"type\"}," + + "{\"type\":\"string\",\"optional\":false,\"field\":\"id\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"string\"," + + "\"optional\":true,\"field\":\"defaultCharsetName\"},{\"type\":\"array\",\"items\":{\"type\":\"string\"," + + "\"optional\":false},\"optional\":true,\"field\":\"primaryKeyColumnNames\"},{\"type\":\"array\"," + + "\"items\":{\"type\":\"struct\",\"fields\":[{\"type\":\"string\",\"optional\":false,\"field\":\"name\"}," + + "{\"type\":\"int32\",\"optional\":false,\"field\":\"jdbcType\"},{\"type\":\"int32\",\"optional\":true," + + "\"field\":\"nativeType\"},{\"type\":\"string\",\"optional\":false,\"field\":\"typeName\"},{\"type\":\"string\"," + + "\"optional\":true,\"field\":\"typeExpression\"},{\"type\":\"string\",\"optional\":true,\"field\":\"charsetName\"}," + + "{\"type\":\"int32\",\"optional\":true,\"field\":\"length\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"scale\"}," + + "{\"type\":\"int32\",\"optional\":false,\"field\":\"position\"},{\"type\":\"boolean\",\"optional\":true," + + "\"field\":\"optional\"},{\"type\":\"boolean\",\"optional\":true,\"field\":\"autoIncremented\"},{\"type\":\"boolean\"," + + "\"optional\":true,\"field\":\"generated\"},{\"type\":\"string\",\"optional\":true,\"field\":\"comment\"}," + + "{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":false},\"optional\":true,\"field\":\"enumValues\"}]," + + "\"optional\":false,\"name\":\"io.debezium.connector.schema.Column\"},\"optional\":false,\"field\":\"columns\"}," + + "{\"type\":\"string\",\"optional\":true,\"field\":\"comment\"}],\"optional\":false," + + "\"name\":\"io.debezium.connector.schema.Table\",\"field\":\"table\"}],\"optional\":false," + + "\"name\":\"io.debezium.connector.schema.Change\"},\"payload\":{\"type\":\"CREATE\"," + + "\"id\":\"\\\"commerce\\\".\\\"orders\\\"\",\"table\":{\"defaultCharsetName\":null,\"primaryKeyColumnNames\":[\"ID\"]," + + "\"columns\":[{\"name\":\"ID\",\"jdbcType\":4,\"nativeType\":null,\"typeName\":\"INTEGER\"," + + "\"typeExpression\":\"INTEGER\",\"charsetName\":null,\"length\":null,\"scale\":null,\"position\":1," + + "\"optional\":false,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"enumValues\":[]}]," + + "\"comment\":null}}}") + .getBytes(StandardCharsets.UTF_8); + + Table restored = VitessTableSchemaState.deserialize(checkpoint); + + assertSchemaFirstIdentity(restored); + Assertions.assertEquals(Collections.singletonList("ID"), restored.primaryKeyColumnNames()); + Assertions.assertEquals(Types.INTEGER, restored.columnWithName("ID").jdbcType()); + Assertions.assertFalse(restored.columnWithName("ID").isOptional()); + + // The namespace correction must not change the existing checkpoint write layout. + try (JsonConverter converter = new JsonConverter()) { + converter.configure(Collections.singletonMap("schemas.enable", true), false); + SchemaAndValue historical = converter.toConnectData("history", checkpoint); + SchemaAndValue rewritten = + converter.toConnectData("history", VitessTableSchemaState.serialize(restored)); + Assertions.assertEquals(historical.schema(), rewritten.schema()); + Assertions.assertEquals(historical.value(), rewritten.value()); + } + } + + private static void assertSchemaFirstIdentity(Table table) { + // TableId.equals compares the combined name, hiding a swapped catalog/schema component. + Assertions.assertNull(table.id().catalog()); + Assertions.assertEquals("commerce", table.id().schema()); + Assertions.assertEquals("orders", table.id().table()); + TableId decoderId = VitessDatabaseSchema.parse("commerce.orders"); + Assertions.assertEquals(decoderId.catalog(), table.id().catalog()); + Assertions.assertEquals(decoderId.schema(), table.id().schema()); + Assertions.assertEquals(decoderId.table(), table.id().table()); + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-db2-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/db2/Db2CDCIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-db2-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/db2/Db2CDCIT.java index fd82f5314a..1f58374070 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-db2-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/db2/Db2CDCIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-db2-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/db2/Db2CDCIT.java @@ -17,6 +17,8 @@ package org.apache.seatunnel.e2e.connector.cdc.db2; +import org.apache.seatunnel.common.utils.SeaTunnelException; +import org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils.TableDiscoveryUtils; import org.apache.seatunnel.e2e.common.TestResource; import org.apache.seatunnel.e2e.common.TestSuiteBase; import org.apache.seatunnel.e2e.common.container.ContainerExtendedFactory; @@ -38,18 +40,23 @@ import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.lifecycle.Startables; import org.testcontainers.utility.DockerLoggerFactory; +import io.debezium.relational.TableId; import lombok.extern.slf4j.Slf4j; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.sql.Types; import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Stream; import static org.awaitility.Awaitility.await; @@ -77,6 +84,12 @@ public class Db2CDCIT extends TestSuiteBase implements TestResource { private static final String SINK_TABLE = SCHEMA + ".CUSTOMERS_SINK"; private static final String CAPTURE_TABLE = "DB2INST1_CUSTOMERS"; private static final String CHANGE_TABLE = CDC_SCHEMA + "." + CAPTURE_TABLE; + private static final String PRODUCT_TABLE = SCHEMA + ".PRODUCTS"; + private static final String PRODUCT_CAPTURE = "DB2INST1_PRODUCTS"; + private static final String PRODUCT_CHANGES = CDC_SCHEMA + "." + PRODUCT_CAPTURE; + private static final String RESTORE_SINK = SCHEMA + ".SAVEPOINT_SINK"; + private static final String MULTI_TABLE_CONF = "/db2cdc_to_db2_with_savepoint.conf"; + private static final String ADD_TABLE_CONF = "/db2cdc_to_db2_add_table_after_savepoint.conf"; private static final String SELECT_CUSTOMERS = "SELECT ID, NAME, DESCRIPTION FROM %s ORDER BY ID"; private static final String SEATUNNEL_SERVER_JVM_OPTION = @@ -136,14 +149,123 @@ public class Db2CDCIT extends TestSuiteBase implements TestResource { @TestTemplate public void testDb2CdcToDb2(TestContainer container) { + runSnapshotAndStream(container, "/db2cdc_to_db2.conf"); + } + + /** Exercises the factory normalization through real catalog discovery and capture filtering. */ + @TestTemplate + public void testLowerCaseTableNames(TestContainer container) throws Exception { + runSnapshotAndStream(container, "/db2cdc_to_db2_with_lower_case_table_names.conf"); + + // Keep the successful uppercase-only path above, then introduce a distinct lowercase + // object. Its uppercase twin's ASN registration must not validate this uncaptured table. + try (Connection connection = getJdbcConnection(); + Statement statement = connection.createStatement()) { + createSchemaIfNotExists(statement, "\"db2inst1\""); + statement.execute( + "CREATE TABLE \"db2inst1\".\"customers\" (ID INTEGER NOT NULL PRIMARY KEY)"); + try { + String resolved = + TableDiscoveryUtils.resolveConfiguredTableName( + "e2e.db2inst1.customers", + path -> { + try (ResultSet tables = + connection + .getMetaData() + .getTables( + null, + path.getSchemaName(), + path.getTableName(), + new String[] {"TABLE"})) { + return tables.next(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + }); + Assertions.assertEquals("e2e.db2inst1.customers", resolved); + List<TableId> captured = new ArrayList<>(); + try (ResultSet registrations = + statement.executeQuery( + "SELECT SOURCE_OWNER, SOURCE_TABLE FROM ASNCDC.IBMSNAP_REGISTER")) { + while (registrations.next()) { + captured.add( + new TableId( + "", + registrations.getString(1), + registrations.getString(2))); + } + } + Assertions.assertTrue(captured.contains(new TableId("", SCHEMA, "CUSTOMERS"))); + SeaTunnelException mismatch = + Assertions.assertThrows( + SeaTunnelException.class, + () -> + TableDiscoveryUtils.validateExplicitCaptureTables( + Collections.singletonList(resolved), captured)); + Assertions.assertTrue(mismatch.getMessage().contains(resolved)); + } finally { + statement.execute("DROP TABLE \"db2inst1\".\"customers\""); + } + } + } + + /** + * Restores both tracked tables, catches up stopped-job writes and resumes both change streams. + * The synthetic capture catalog does not exercise IBM ASN DDL-history replay. + */ + @TestTemplate + public void testMultiTableSavepointRestore(TestContainer container) throws Exception { + initializeDb2Tables(true); + CompletableFuture<Void> initial = executeAsync(container, MULTI_TABLE_CONF, null); + awaitRestoreTables(initial, true); + applyPhase(0, true); + awaitRestoreTables(initial, true); + String jobId = stopAtSavepoint(container, MULTI_TABLE_CONF, initial); + + applyPhase(1, true); + CompletableFuture<Void> restored = executeAsync(container, MULTI_TABLE_CONF, jobId); + awaitRestoreTables(restored, true); + applyPhase(2, true); + awaitRestoreTables(restored, true); + } + + /** + * Adds and registers the second source table only after the original single-table job stops. + */ + @TestTemplate + public void testAddTableDuringSavepointRestore(TestContainer container) throws Exception { + initializeDb2Tables(false); + CompletableFuture<Void> initial = executeAsync(container, ADD_TABLE_CONF, null); + awaitRestoreTables(initial, false); + applyPhase(0, false); + awaitRestoreTables(initial, false); + String jobId = stopAtSavepoint(container, ADD_TABLE_CONF, initial); + + applyPhase(1, false); + try (Connection connection = getJdbcConnection(); + Statement statement = connection.createStatement()) { + createProductTables(connection, statement); + } + CompletableFuture<Void> restored = executeAsync(container, MULTI_TABLE_CONF, jobId); + awaitRestoreTables(restored, true); + applyPhase(2, false); + try (Connection connection = getJdbcConnection()) { + connection.setAutoCommit(false); + applyTableChanges(connection, PRODUCT_TABLE, PRODUCT_CHANGES, 11, 13, 11); + updateCaptureSynchpoint(connection, "PRODUCTS", 12); + connection.commit(); + } + awaitRestoreTables(restored, true); + } + + private void runSnapshotAndStream(TestContainer container, String configuration) { initializeDb2Tables(); CompletableFuture<Void> jobFuture = CompletableFuture.runAsync( () -> { try { - Container.ExecResult result = - container.executeJob("/db2cdc_to_db2.conf"); + Container.ExecResult result = container.executeJob(configuration); Assertions.assertEquals( 0, result.getExitCode(), result.getStderr()); } catch (Exception e) { @@ -173,11 +295,18 @@ public class Db2CDCIT extends TestSuiteBase implements TestResource { } private void initializeDb2Tables() { + initializeDb2Tables(false); + } + + private void initializeDb2Tables(boolean includeProducts) { try (Connection connection = getJdbcConnection(); Statement statement = connection.createStatement()) { createSchemaIfNotExists(statement, SCHEMA); createSchemaIfNotExists(statement, CDC_SCHEMA); dropTableIfExists(statement, CHANGE_TABLE); + dropTableIfExists(statement, PRODUCT_CHANGES); + dropTableIfExists(statement, PRODUCT_TABLE); + dropTableIfExists(statement, RESTORE_SINK); dropTableIfExists(statement, CDC_SCHEMA + ".IBMSNAP_REGISTER"); dropTableIfExists(statement, SINK_TABLE); dropTableIfExists(statement, SOURCE_TABLE); @@ -218,6 +347,13 @@ public class Db2CDCIT extends TestSuiteBase implements TestResource { insertCustomer(connection, SOURCE_TABLE, 1, "Alice", "snapshot row 1"); insertCustomer(connection, SOURCE_TABLE, 2, "Bob", "snapshot row 2"); insertCaptureRegistration(connection); + statement.execute( + "CREATE TABLE " + + RESTORE_SINK + + " (ID INTEGER NOT NULL, NAME VARCHAR(64) NOT NULL, DESCRIPTION VARCHAR(128), PRIMARY KEY (ID))"); + if (includeProducts) { + createProductTables(connection, statement); + } } catch (SQLException e) { throw new RuntimeException("Failed to initialize DB2 CDC E2E tables", e); } @@ -259,6 +395,11 @@ public class Db2CDCIT extends TestSuiteBase implements TestResource { } private void insertCaptureRegistration(Connection connection) throws SQLException { + insertCaptureRegistration(connection, "CUSTOMERS", CAPTURE_TABLE); + } + + private void insertCaptureRegistration(Connection connection, String table, String capture) + throws SQLException { try (PreparedStatement statement = connection.prepareStatement( "INSERT INTO " @@ -268,30 +409,36 @@ public class Db2CDCIT extends TestSuiteBase implements TestResource { + "CD_NEW_SYNCHPOINT, CD_OLD_SYNCHPOINT, SYNCHPOINT) " + "VALUES (?, ?, ?, ?, ?, ?, ?)")) { statement.setString(1, SCHEMA); - statement.setString(2, "CUSTOMERS"); + statement.setString(2, table); statement.setString(3, CDC_SCHEMA); - statement.setString(4, CAPTURE_TABLE); + statement.setString(4, capture); statement.setBytes(5, lsn(0)); - // A single active capture instance must not have a stop LSN. Debezium treats a - // non-null CD_OLD_SYNCHPOINT as the stop point of a retired capture table. - statement.setNull(6, Types.BINARY); + // A real active registration can have a non-null OLD point. Treating it as a + // stop LSN would filter every subsequent change and fail the source/sink checks. + statement.setBytes(6, lsn(0)); statement.setBytes(7, lsn(0)); statement.executeUpdate(); } } private void updateCaptureSynchpoint(Connection connection, int commitLsn) throws SQLException { + updateCaptureSynchpoint(connection, "CUSTOMERS", commitLsn); + } + + private void updateCaptureSynchpoint(Connection connection, String table, int commitLsn) + throws SQLException { try (PreparedStatement statement = connection.prepareStatement( "UPDATE " + CDC_SCHEMA - + ".IBMSNAP_REGISTER SET SYNCHPOINT = ? " + + ".IBMSNAP_REGISTER SET SYNCHPOINT = ?, CD_NEW_SYNCHPOINT = ? " + "WHERE SOURCE_OWNER = ? AND SOURCE_TABLE = ?")) { // Debezium polls the max LSN from IBMSNAP_REGISTER, so advancing SYNCHPOINT makes the // synthetic capture rows visible to the transaction-log reader. statement.setBytes(1, lsn(commitLsn)); - statement.setString(2, SCHEMA); - statement.setString(3, "CUSTOMERS"); + statement.setBytes(2, lsn(commitLsn)); + statement.setString(3, SCHEMA); + statement.setString(4, table); statement.executeUpdate(); } } @@ -329,10 +476,22 @@ public class Db2CDCIT extends TestSuiteBase implements TestResource { String name, String description) throws SQLException { + insertChangeEvent(connection, CHANGE_TABLE, commitLsn, operation, id, name, description); + } + + private void insertChangeEvent( + Connection connection, + String changeTable, + int commitLsn, + String operation, + int id, + String name, + String description) + throws SQLException { try (PreparedStatement statement = connection.prepareStatement( "INSERT INTO " - + CHANGE_TABLE + + changeTable + " (IBMSNAP_COMMITSEQ, IBMSNAP_INTENTSEQ, IBMSNAP_OPERATION, " + "ID, NAME, DESCRIPTION) " + "VALUES (?, ?, ?, ?, ?, ?)")) { @@ -346,6 +505,143 @@ public class Db2CDCIT extends TestSuiteBase implements TestResource { } } + private void createProductTables(Connection connection, Statement statement) + throws SQLException { + statement.execute( + "CREATE TABLE " + + PRODUCT_TABLE + + " (ID INTEGER NOT NULL, NAME VARCHAR(64) NOT NULL, DESCRIPTION VARCHAR(128), PRIMARY KEY (ID))"); + statement.execute( + "CREATE TABLE " + + PRODUCT_CHANGES + + " (IBMSNAP_COMMITSEQ CHAR(16) FOR BIT DATA NOT NULL, IBMSNAP_INTENTSEQ CHAR(16) FOR BIT DATA NOT NULL, " + + "IBMSNAP_OPERATION CHAR(1) NOT NULL, ID INTEGER, NAME VARCHAR(64), DESCRIPTION VARCHAR(128))"); + insertCustomer(connection, PRODUCT_TABLE, 11, "Keyboard", "snapshot product 1"); + insertCustomer(connection, PRODUCT_TABLE, 12, "Mouse", "snapshot product 2"); + insertCaptureRegistration(connection, "PRODUCTS", PRODUCT_CAPTURE); + } + + /** + * Commits deterministic insert/delete pairs at distinct LSNs for each table and recovery phase. + */ + private void applyPhase(int phase, boolean includeProducts) { + try (Connection connection = getJdbcConnection()) { + connection.setAutoCommit(false); + int firstLsn = phase * 4 + 1; + applyTableChanges( + connection, SOURCE_TABLE, CHANGE_TABLE, firstLsn, phase + 3, phase + 1); + updateCaptureSynchpoint(connection, "CUSTOMERS", firstLsn + 1); + if (includeProducts) { + applyTableChanges( + connection, + PRODUCT_TABLE, + PRODUCT_CHANGES, + firstLsn + 2, + phase + 13, + phase + 11); + updateCaptureSynchpoint(connection, "PRODUCTS", firstLsn + 3); + } + connection.commit(); + } catch (SQLException e) { + throw new RuntimeException("Failed to write DB2 recovery phase " + phase, e); + } + } + + private void applyTableChanges( + Connection connection, + String table, + String changeTable, + int firstLsn, + int insertedId, + int deletedId) + throws SQLException { + String name = "row-" + insertedId; + String description = "stream-" + firstLsn; + insertCustomer(connection, table, insertedId, name, description); + insertChangeEvent(connection, changeTable, firstLsn, "I", insertedId, name, description); + try (PreparedStatement query = + connection.prepareStatement( + "SELECT NAME, DESCRIPTION FROM " + table + " WHERE ID = ?")) { + query.setInt(1, deletedId); + try (ResultSet row = query.executeQuery()) { + Assertions.assertTrue( + row.next(), "Delete fixture must exist: " + table + "/" + deletedId); + insertChangeEvent( + connection, + changeTable, + firstLsn + 1, + "D", + deletedId, + row.getString(1), + row.getString(2)); + } + } + deleteCustomer(connection, table, deletedId); + } + + private CompletableFuture<Void> executeAsync( + TestContainer container, String configuration, String restoreJobId) { + return CompletableFuture.runAsync( + () -> { + try { + Container.ExecResult result = + restoreJobId == null + ? container.executeJob(configuration) + : container.restoreJob(configuration, restoreJobId); + Assertions.assertEquals(0, result.getExitCode(), result.getStderr()); + } catch (Exception e) { + throw new RuntimeException("Failed to run DB2 CDC recovery job", e); + } + }); + } + + private void awaitRestoreTables(CompletableFuture<Void> job, boolean includeProducts) { + await().atMost(2, TimeUnit.MINUTES) + .untilAsserted( + () -> { + assertJobNotFailed(job); + String source = + includeProducts + ? "(SELECT ID, NAME, DESCRIPTION FROM " + + SOURCE_TABLE + + " UNION ALL SELECT ID, NAME, DESCRIPTION FROM " + + PRODUCT_TABLE + + ") AS SOURCES" + : SOURCE_TABLE; + Assertions.assertIterableEquals( + querySql(SELECT_CUSTOMERS, source), + querySql(SELECT_CUSTOMERS, RESTORE_SINK)); + }); + } + + /** + * The Apache test runner supplies --name using the configuration basename, overriding + * env.job.name. + */ + private String stopAtSavepoint( + TestContainer container, String configuration, CompletableFuture<Void> initial) + throws Exception { + String jobName = configuration.substring(configuration.lastIndexOf('/') + 1); + Matcher matcher = + Pattern.compile( + "Init JobMaster for Job " + + Pattern.quote(jobName) + + " \\(([0-9]+)\\)") + .matcher(container.getServerLogs()); + String jobId = null; + int matches = 0; + while (matcher.find()) { + jobId = matcher.group(1); + matches++; + } + Assertions.assertEquals(1, matches, "Expected one initial DB2 CDC job for " + jobName); + Container.ExecResult result = container.savepointJob(jobId); + Assertions.assertEquals(0, result.getExitCode(), result.getStderr()); + await().atMost(30, TimeUnit.SECONDS).until(initial::isDone); + assertJobNotFailed(initial); + return jobId; + } + private Connection getJdbcConnection() throws SQLException { return DriverManager.getConnection( DB2_CONTAINER.getJdbcUrl(), diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-db2-e2e/src/test/resources/db2cdc_to_db2_add_table_after_savepoint.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-db2-e2e/src/test/resources/db2cdc_to_db2_add_table_after_savepoint.conf new file mode 100644 index 0000000000..4a42dd4832 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-db2-e2e/src/test/resources/db2cdc_to_db2_add_table_after_savepoint.conf @@ -0,0 +1,51 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "STREAMING" + checkpoint.interval = 5000 +} + +source { + DB2-CDC { + plugin_output = "customers" + username = "db2inst1" + password = "123456" + startup.mode = "initial" + database-names = ["E2E"] + table-names = ["E2E.DB2INST1.CUSTOMERS"] + url = "jdbc:db2://db2-cdc-e2e:50000/E2E" + snapshot.split.size = 2 + } +} + +sink { + Jdbc { + plugin_input = "customers" + driver = "com.ibm.db2.jcc.DB2Driver" + url = "jdbc:db2://db2-cdc-e2e:50000/E2E" + username = "db2inst1" + password = "123456" + generate_sink_sql = true + database = "E2E" + schema = "DB2INST1" + table = "SAVEPOINT_SINK" + batch_size = 1 + primary_keys = ["ID"] + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-db2-e2e/src/test/resources/db2cdc_to_db2_with_lower_case_table_names.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-db2-e2e/src/test/resources/db2cdc_to_db2_with_lower_case_table_names.conf new file mode 100644 index 0000000000..02f51c3eb5 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-db2-e2e/src/test/resources/db2cdc_to_db2_with_lower_case_table_names.conf @@ -0,0 +1,51 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "STREAMING" + checkpoint.interval = 5000 +} + +source { + DB2-CDC { + plugin_output = "customers" + username = "db2inst1" + password = "123456" + startup.mode = "initial" + database-names = ["e2e"] + table-names = ["e2e.db2inst1.customers"] + url = "jdbc:db2://db2-cdc-e2e:50000/e2e" + snapshot.split.size = 2 + } +} + +sink { + Jdbc { + plugin_input = "customers" + driver = "com.ibm.db2.jcc.DB2Driver" + url = "jdbc:db2://db2-cdc-e2e:50000/E2E" + username = "db2inst1" + password = "123456" + generate_sink_sql = true + database = "E2E" + schema = "DB2INST1" + table = "CUSTOMERS_SINK" + batch_size = 1 + primary_keys = ["ID"] + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-db2-e2e/src/test/resources/db2cdc_to_db2_with_savepoint.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-db2-e2e/src/test/resources/db2cdc_to_db2_with_savepoint.conf new file mode 100644 index 0000000000..dcdc1552ac --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-db2-e2e/src/test/resources/db2cdc_to_db2_with_savepoint.conf @@ -0,0 +1,51 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "STREAMING" + checkpoint.interval = 5000 +} + +source { + DB2-CDC { + plugin_output = "customers" + username = "db2inst1" + password = "123456" + startup.mode = "initial" + database-names = ["E2E"] + table-names = ["E2E.DB2INST1.CUSTOMERS", "E2E.DB2INST1.PRODUCTS"] + url = "jdbc:db2://db2-cdc-e2e:50000/E2E" + snapshot.split.size = 2 + } +} + +sink { + Jdbc { + plugin_input = "customers" + driver = "com.ibm.db2.jcc.DB2Driver" + url = "jdbc:db2://db2-cdc-e2e:50000/E2E" + username = "db2inst1" + password = "123456" + generate_sink_sql = true + database = "E2E" + schema = "DB2INST1" + table = "SAVEPOINT_SINK" + batch_size = 1 + primary_keys = ["ID"] + } +}
