This is an automated email from the ASF dual-hosted git repository. kunni pushed a commit to branch release-3.6 in repository https://gitbox.apache.org/repos/asf/flink-cdc.git
commit f27743b355b314aedd82d00e0ee193b5b4fdfe53 Author: Kunni <[email protected]> AuthorDate: Thu Mar 26 09:57:39 2026 +0800 [FLINK-39288][docs] Update DataStream API package guidance for Flink 1.20.x and 2.2.x (#4339) Co-authored-by: Claude Opus 4.6 <[email protected]> (cherry picked from commit 619ab90477c9c4ac6752e3ad4eb0800bcd14023b) --- .../datastream-api-package-guidance.md | 236 ++++++++++++++++++++- .../datastream-api-package-guidance.md | 236 ++++++++++++++++++++- 2 files changed, 452 insertions(+), 20 deletions(-) diff --git a/docs/content.zh/docs/connectors/flink-sources/datastream-api-package-guidance.md b/docs/content.zh/docs/connectors/flink-sources/datastream-api-package-guidance.md index ccd117160..a38e4c4fa 100644 --- a/docs/content.zh/docs/connectors/flink-sources/datastream-api-package-guidance.md +++ b/docs/content.zh/docs/connectors/flink-sources/datastream-api-package-guidance.md @@ -29,7 +29,7 @@ under the License. This guide provides a simple `pom.xml` example for packaging DataStream job JARs with MySQL CDC source. -## Example for `pom.xml` +## Example for `pom.xml` with Flink 1.20.x ```xml <?xml version="1.0" encoding="UTF-8"?> @@ -43,23 +43,21 @@ This guide provides a simple `pom.xml` example for packaging DataStream job JARs <version>1.0-SNAPSHOT</version> <properties> - <maven.compiler.source>8</maven.compiler.source> - <maven.compiler.target>8</maven.compiler.target> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <java.version>1.8</java.version> - <scala.binary.version>2.12</scala.binary.version> + <java.version>11</java.version> <maven.compiler.source>${java.version}</maven.compiler.source> <maven.compiler.target>${java.version}</maven.compiler.target> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <scala.binary.version>2.12</scala.binary.version> <!-- Enforce single fork execution due to heavy mini cluster use in the tests --> <flink.forkCount>1</flink.forkCount> <flink.reuseForks>true</flink.reuseForks> <!-- dependencies versions --> - <flink.version>1.17.2</flink.version> + <flink.version>1.20.3</flink.version> <slf4j.version>1.7.15</slf4j.version> <log4j.version>2.17.1</log4j.version> <debezium.version>1.9.7.Final</debezium.version> + <flink.cdc.version>3.6.0-1.20</flink.cdc.version> </properties> <dependencies> <dependency> @@ -114,12 +112,230 @@ This guide provides a simple `pom.xml` example for packaging DataStream job JARs <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-shaded-guava</artifactId> - <version>30.1.1-jre-16.1</version> + <version>31.1-jre-17.0</version> </dependency> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-connector-base</artifactId> + <version>${flink.version}</version> + <scope>provided</scope> + </dependency> + <!-- Logging --> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-slf4j-impl</artifactId> + <version>${log4j.version}</version> + <scope>provided</scope> + </dependency> + <!-- Checked the dependencies of the Flink project and below is a feasible reference. --> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-connector-mysql-cdc</artifactId> - <version>2.4.2</version> + <version>${flink.cdc.version}</version> + </dependency> + <dependency> + <groupId>io.debezium</groupId> + <artifactId>debezium-connector-mysql</artifactId> + <version>${debezium.version}</version> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <executions> + <execution> + <id>shade-flink</id> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <!-- Shading test jar have bug in some previous version, so close this configuration here, + see https://issues.apache.org/jira/browse/MSHADE-284 --> + <shadeTestJar>false</shadeTestJar> + <shadedArtifactAttached>false</shadedArtifactAttached> + <createDependencyReducedPom>true</createDependencyReducedPom> + <dependencyReducedPomLocation> + ${project.basedir}/target/dependency-reduced-pom.xml + </dependencyReducedPomLocation> + <filters combine.children="append"> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>module-info.class</exclude> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> + <artifactSet> + <includes> + <!-- include nothing --> + <include>io.debezium:debezium-api</include> + <include>io.debezium:debezium-embedded</include> + <include>io.debezium:debezium-core</include> + <include>io.debezium:debezium-ddl-parser</include> + <include>io.debezium:debezium-connector-mysql</include> + <include>org.apache.flink:flink-connector-debezium</include> + <include>org.apache.flink:flink-sql-connector-mysql-cdc</include> + <include>org.antlr:antlr4-runtime</include> + <include>org.apache.kafka:*</include> + <include>mysql:mysql-connector-java</include> + <include>com.zendesk:mysql-binlog-connector-java</include> + <include>com.fasterxml.*:*</include> + <include>com.google.guava:*</include> + <include>com.esri.geometry:esri-geometry-api</include> + <include>com.zaxxer:HikariCP</include> + <include>org.apache.flink:flink-shaded-guava</include> + </includes> + </artifactSet> + <relocations> + <relocation> + <pattern>org.apache.kafka</pattern> + <shadedPattern> + org.apache.flink.cdc.connectors.shaded.org.apache.kafka + </shadedPattern> + </relocation> + <relocation> + <pattern>org.antlr</pattern> + <shadedPattern> + org.apache.flink.cdc.connectors.shaded.org.antlr + </shadedPattern> + </relocation> + <relocation> + <pattern>com.fasterxml</pattern> + <shadedPattern> + org.apache.flink.cdc.connectors.shaded.com.fasterxml + </shadedPattern> + </relocation> + <relocation> + <pattern>com.google</pattern> + <shadedPattern> + org.apache.flink.cdc.connectors.shaded.com.google + </shadedPattern> + </relocation> + <relocation> + <pattern>com.esri.geometry</pattern> + <shadedPattern>org.apache.flink.cdc.connectors.shaded.com.esri.geometry</shadedPattern> + </relocation> + <relocation> + <pattern>com.zaxxer</pattern> + <shadedPattern> + org.apache.flink.cdc.connectors.shaded.com.zaxxer + </shadedPattern> + </relocation> + </relocations> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> +``` + +## Example for `pom.xml` with Flink 2.2.x + +```xml +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <groupId>org.apache.flink</groupId> + <artifactId>FlinkCDCTest</artifactId> + <version>1.0-SNAPSHOT</version> + + <properties> + <java.version>11</java.version> + <maven.compiler.source>${java.version}</maven.compiler.source> + <maven.compiler.target>${java.version}</maven.compiler.target> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <scala.binary.version>2.12</scala.binary.version> + <!-- Enforce single fork execution due to heavy mini cluster use in the tests --> + <flink.forkCount>1</flink.forkCount> + <flink.reuseForks>true</flink.reuseForks> + + <!-- dependencies versions --> + <flink.version>2.2.0</flink.version> + <slf4j.version>1.7.15</slf4j.version> + <log4j.version>2.17.1</log4j.version> + <debezium.version>1.9.7.Final</debezium.version> + <flink.cdc.version>3.6.0-2.2</flink.cdc.version> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-streaming-java</artifactId> + <version>${flink.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-clients</artifactId> + <version>${flink.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-table-planner_${scala.binary.version}</artifactId> + <version>${flink.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-table-runtime</artifactId> + <version>${flink.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-core</artifactId> + <version>${flink.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-table-common</artifactId> + <version>${flink.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-connector-base</artifactId> + <version>${flink.version}</version> + <scope>provided</scope> + </dependency> + <!-- Logging --> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-slf4j-impl</artifactId> + <version>${log4j.version}</version> + <scope>provided</scope> + </dependency> + <!-- Checked the dependencies of the Flink project and below is a feasible reference. --> + <!-- Checked the dependencies of the Flink project and below is a feasible reference. --> + <!-- Use flink shaded guava 18.0-13.0 for flink 1.13 --> + <!-- Use flink shaded guava 30.1.1-jre-14.0 for flink-1.14 --> + <!-- Use flink shaded guava 30.1.1-jre-15.0 for flink-1.15 --> + <!-- Use flink shaded guava 30.1.1-jre-15.0 for flink-1.16 --> + <!-- Use flink shaded guava 30.1.1-jre-16.1 for flink-1.17 --> + <!-- Use flink shaded guava 31.1-jre-17.0 for flink-1.18 --> + <!-- Use flink shaded guava 33.4.0-jre-20.0 for flink-2.2 --> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-shaded-guava</artifactId> + <version>33.4.0-jre-20.0</version> + </dependency> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-sql-connector-mysql-cdc</artifactId> + <version>${flink.cdc.version}</version> </dependency> <dependency> <groupId>io.debezium</groupId> @@ -169,7 +385,7 @@ This guide provides a simple `pom.xml` example for packaging DataStream job JARs <include>io.debezium:debezium-ddl-parser</include> <include>io.debezium:debezium-connector-mysql</include> <include>org.apache.flink:flink-connector-debezium</include> - <include>org.apache.flink:flink-connector-mysql-cdc</include> + <include>org.apache.flink:flink-sql-connector-mysql-cdc</include> <include>org.antlr:antlr4-runtime</include> <include>org.apache.kafka:*</include> <include>mysql:mysql-connector-java</include> diff --git a/docs/content/docs/connectors/flink-sources/datastream-api-package-guidance.md b/docs/content/docs/connectors/flink-sources/datastream-api-package-guidance.md index 7569fe6e9..e2f608130 100644 --- a/docs/content/docs/connectors/flink-sources/datastream-api-package-guidance.md +++ b/docs/content/docs/connectors/flink-sources/datastream-api-package-guidance.md @@ -29,7 +29,7 @@ under the License. This guide provides a simple `pom.xml` example for packaging DataStream job JARs with MySQL CDC source. -## Example for `pom.xml` +## Example for `pom.xml` with Flink 1.20.x ```xml <?xml version="1.0" encoding="UTF-8"?> @@ -43,23 +43,21 @@ This guide provides a simple `pom.xml` example for packaging DataStream job JARs <version>1.0-SNAPSHOT</version> <properties> - <maven.compiler.source>8</maven.compiler.source> - <maven.compiler.target>8</maven.compiler.target> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <java.version>1.8</java.version> - <scala.binary.version>2.12</scala.binary.version> + <java.version>11</java.version> <maven.compiler.source>${java.version}</maven.compiler.source> <maven.compiler.target>${java.version}</maven.compiler.target> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <scala.binary.version>2.12</scala.binary.version> <!-- Enforce single fork execution due to heavy mini cluster use in the tests --> <flink.forkCount>1</flink.forkCount> <flink.reuseForks>true</flink.reuseForks> <!-- dependencies versions --> - <flink.version>1.17.2</flink.version> + <flink.version>1.20.3</flink.version> <slf4j.version>1.7.15</slf4j.version> <log4j.version>2.17.1</log4j.version> <debezium.version>1.9.7.Final</debezium.version> + <flink.cdc.version>3.6.0-1.20</flink.cdc.version> </properties> <dependencies> <dependency> @@ -114,12 +112,230 @@ This guide provides a simple `pom.xml` example for packaging DataStream job JARs <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-shaded-guava</artifactId> - <version>30.1.1-jre-16.1</version> + <version>31.1-jre-17.0</version> </dependency> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-connector-base</artifactId> + <version>${flink.version}</version> + <scope>provided</scope> + </dependency> + <!-- Logging --> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-slf4j-impl</artifactId> + <version>${log4j.version}</version> + <scope>provided</scope> + </dependency> + <!-- Checked the dependencies of the Flink project and below is a feasible reference. --> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-connector-mysql-cdc</artifactId> - <version>2.4.2</version> + <version>${flink.cdc.version}</version> + </dependency> + <dependency> + <groupId>io.debezium</groupId> + <artifactId>debezium-connector-mysql</artifactId> + <version>${debezium.version}</version> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <executions> + <execution> + <id>shade-flink</id> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <!-- Shading test jar have bug in some previous version, so close this configuration here, + see https://issues.apache.org/jira/browse/MSHADE-284 --> + <shadeTestJar>false</shadeTestJar> + <shadedArtifactAttached>false</shadedArtifactAttached> + <createDependencyReducedPom>true</createDependencyReducedPom> + <dependencyReducedPomLocation> + ${project.basedir}/target/dependency-reduced-pom.xml + </dependencyReducedPomLocation> + <filters combine.children="append"> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>module-info.class</exclude> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> + <artifactSet> + <includes> + <!-- include nothing --> + <include>io.debezium:debezium-api</include> + <include>io.debezium:debezium-embedded</include> + <include>io.debezium:debezium-core</include> + <include>io.debezium:debezium-ddl-parser</include> + <include>io.debezium:debezium-connector-mysql</include> + <include>org.apache.flink:flink-connector-debezium</include> + <include>org.apache.flink:flink-sql-connector-mysql-cdc</include> + <include>org.antlr:antlr4-runtime</include> + <include>org.apache.kafka:*</include> + <include>mysql:mysql-connector-java</include> + <include>com.zendesk:mysql-binlog-connector-java</include> + <include>com.fasterxml.*:*</include> + <include>com.google.guava:*</include> + <include>com.esri.geometry:esri-geometry-api</include> + <include>com.zaxxer:HikariCP</include> + <include>org.apache.flink:flink-shaded-guava</include> + </includes> + </artifactSet> + <relocations> + <relocation> + <pattern>org.apache.kafka</pattern> + <shadedPattern> + org.apache.flink.cdc.connectors.shaded.org.apache.kafka + </shadedPattern> + </relocation> + <relocation> + <pattern>org.antlr</pattern> + <shadedPattern> + org.apache.flink.cdc.connectors.shaded.org.antlr + </shadedPattern> + </relocation> + <relocation> + <pattern>com.fasterxml</pattern> + <shadedPattern> + org.apache.flink.cdc.connectors.shaded.com.fasterxml + </shadedPattern> + </relocation> + <relocation> + <pattern>com.google</pattern> + <shadedPattern> + org.apache.flink.cdc.connectors.shaded.com.google + </shadedPattern> + </relocation> + <relocation> + <pattern>com.esri.geometry</pattern> + <shadedPattern>org.apache.flink.cdc.connectors.shaded.com.esri.geometry</shadedPattern> + </relocation> + <relocation> + <pattern>com.zaxxer</pattern> + <shadedPattern> + org.apache.flink.cdc.connectors.shaded.com.zaxxer + </shadedPattern> + </relocation> + </relocations> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> +``` + +## Example for `pom.xml` with Flink 2.2.x + +```xml +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <groupId>org.apache.flink</groupId> + <artifactId>FlinkCDCTest</artifactId> + <version>1.0-SNAPSHOT</version> + + <properties> + <java.version>11</java.version> + <maven.compiler.source>${java.version}</maven.compiler.source> + <maven.compiler.target>${java.version}</maven.compiler.target> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <scala.binary.version>2.12</scala.binary.version> + <!-- Enforce single fork execution due to heavy mini cluster use in the tests --> + <flink.forkCount>1</flink.forkCount> + <flink.reuseForks>true</flink.reuseForks> + + <!-- dependencies versions --> + <flink.version>2.2.0</flink.version> + <slf4j.version>1.7.15</slf4j.version> + <log4j.version>2.17.1</log4j.version> + <debezium.version>1.9.7.Final</debezium.version> + <flink.cdc.version>3.6.0-2.2</flink.cdc.version> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-streaming-java</artifactId> + <version>${flink.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-clients</artifactId> + <version>${flink.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-table-planner_${scala.binary.version}</artifactId> + <version>${flink.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-table-runtime</artifactId> + <version>${flink.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-core</artifactId> + <version>${flink.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-table-common</artifactId> + <version>${flink.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-connector-base</artifactId> + <version>${flink.version}</version> + <scope>provided</scope> + </dependency> + <!-- Logging --> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-slf4j-impl</artifactId> + <version>${log4j.version}</version> + <scope>provided</scope> + </dependency> + <!-- Checked the dependencies of the Flink project and below is a feasible reference. --> + <!-- Checked the dependencies of the Flink project and below is a feasible reference. --> + <!-- Use flink shaded guava 18.0-13.0 for flink 1.13 --> + <!-- Use flink shaded guava 30.1.1-jre-14.0 for flink-1.14 --> + <!-- Use flink shaded guava 30.1.1-jre-15.0 for flink-1.15 --> + <!-- Use flink shaded guava 30.1.1-jre-15.0 for flink-1.16 --> + <!-- Use flink shaded guava 30.1.1-jre-16.1 for flink-1.17 --> + <!-- Use flink shaded guava 31.1-jre-17.0 for flink-1.18 --> + <!-- Use flink shaded guava 33.4.0-jre-20.0 for flink-2.2 --> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-shaded-guava</artifactId> + <version>33.4.0-jre-20.0</version> + </dependency> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-sql-connector-mysql-cdc</artifactId> + <version>${flink.cdc.version}</version> </dependency> <dependency> <groupId>io.debezium</groupId> @@ -169,7 +385,7 @@ This guide provides a simple `pom.xml` example for packaging DataStream job JARs <include>io.debezium:debezium-ddl-parser</include> <include>io.debezium:debezium-connector-mysql</include> <include>org.apache.flink:flink-connector-debezium</include> - <include>org.apache.flink:flink-connector-mysql-cdc</include> + <include>org.apache.flink:flink-sql-connector-mysql-cdc</include> <include>org.antlr:antlr4-runtime</include> <include>org.apache.kafka:*</include> <include>mysql:mysql-connector-java</include>
