This is an automated email from the ASF dual-hosted git repository. danderson pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/flink-playgrounds.git
The following commit(s) were added to refs/heads/master by this push: new a2b62f2 FLINK-30442: Upgrading table walk-through to Flink 1.16 a2b62f2 is described below commit a2b62f25c7647f5a35e672a9e508bb5c80424977 Author: Gunnar Morling <gunnar.morl...@googlemail.com> AuthorDate: Tue Jan 3 14:30:41 2023 +0100 FLINK-30442: Upgrading table walk-through to Flink 1.16 * [FLINK-30440] Updating README * [FLINK-30442] Misc. version updates for table walk-through; - Java 17 for data generator, Java 11 for Flink job - Kafka client - Also removing some dead code * [FLINK-30422] Upgrading table walk-through to Flink 1.16 --- docker/data-generator/Dockerfile | 6 ++--- docker/data-generator/pom.xml | 26 +++++++++++++--------- .../flink/playground/datagen/DataGenerator.java | 7 ++++-- .../apache/flink/playground/datagen/Producer.java | 4 ---- docker/ops-playground-image/Dockerfile | 2 +- .../java/flink-playground-clickcountjob/pom.xml | 14 ------------ operations-playground/README.md | 2 +- table-walkthrough/Dockerfile | 12 +++++----- table-walkthrough/docker-compose.yml | 4 ++-- table-walkthrough/pom.xml | 18 ++------------- 10 files changed, 35 insertions(+), 60 deletions(-) diff --git a/docker/data-generator/Dockerfile b/docker/data-generator/Dockerfile index 124033c..1745a2d 100644 --- a/docker/data-generator/Dockerfile +++ b/docker/data-generator/Dockerfile @@ -13,16 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM maven:3.8-jdk-8-slim AS builder +FROM maven:3.8-eclipse-temurin-17 AS builder # Get data producer code and compile it COPY ./src /opt/data-producer/src COPY ./pom.xml /opt/data-producer/pom.xml RUN cd /opt/data-producer; \ - mvn clean install + mvn clean verify -FROM openjdk:8-jre +FROM eclipse-temurin:17 COPY --from=builder /opt/data-producer/target/data-generator-*.jar /opt/data-generator.jar diff --git a/docker/data-generator/pom.xml b/docker/data-generator/pom.xml index 95ced48..ae9a683 100644 --- a/docker/data-generator/pom.xml +++ b/docker/data-generator/pom.xml @@ -39,13 +39,11 @@ under the License. <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <kafka.version>2.2.0</kafka.version> - <avro.version>1.8.2</avro.version> - <java.version>1.8</java.version> - <java.version>1.8</java.version> + <kafka.version>3.3.1</kafka.version> + <slf4j.version>1.7.36</slf4j.version> + <java.version>17</java.version> <spotless-maven-plugin.version>1.20.0</spotless-maven-plugin.version> - <maven.compiler.source>${java.version}</maven.compiler.source> - <maven.compiler.target>${java.version}</maven.compiler.target> + <maven.compiler.release>${java.version}</maven.compiler.release> </properties> <dependencies> @@ -54,6 +52,16 @@ under the License. <artifactId>kafka-clients</artifactId> <version>${kafka.version}</version> </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <version>${slf4j.version}</version> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-simple</artifactId> + <version>${slf4j.version}</version> + </dependency> </dependencies> <build> @@ -62,11 +70,7 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <version>3.1</version> - <configuration> - <source>${java.version}</source> - <target>${java.version}</target> - </configuration> + <version>3.10.1</version> </plugin> <!-- Shade plugin to include all dependencies --> diff --git a/docker/data-generator/src/main/java/org/apache/flink/playground/datagen/DataGenerator.java b/docker/data-generator/src/main/java/org/apache/flink/playground/datagen/DataGenerator.java index 250f2c5..d66b0f7 100644 --- a/docker/data-generator/src/main/java/org/apache/flink/playground/datagen/DataGenerator.java +++ b/docker/data-generator/src/main/java/org/apache/flink/playground/datagen/DataGenerator.java @@ -18,6 +18,7 @@ package org.apache.flink.playground.datagen; +import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,9 +27,11 @@ public class DataGenerator { private static final Logger LOG = LoggerFactory.getLogger(DataGenerator.class); - private static final String KAFKA = "kafka:9092"; + private static final String KAFKA = + Optional.ofNullable(System.getenv("DATAGEN_KAFKA")).orElse("kafka:9092"); - private static final String TOPIC = "transactions"; + private static final String TOPIC = + Optional.ofNullable(System.getenv("DATAGEN_TOPIC")).orElse("transactions"); public static void main(String[] args) { Producer producer = new Producer(KAFKA, TOPIC); diff --git a/docker/data-generator/src/main/java/org/apache/flink/playground/datagen/Producer.java b/docker/data-generator/src/main/java/org/apache/flink/playground/datagen/Producer.java index 33d68bd..b127b0f 100644 --- a/docker/data-generator/src/main/java/org/apache/flink/playground/datagen/Producer.java +++ b/docker/data-generator/src/main/java/org/apache/flink/playground/datagen/Producer.java @@ -19,7 +19,6 @@ package org.apache.flink.playground.datagen; import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; import java.util.Properties; import org.apache.flink.playground.datagen.model.Transaction; import org.apache.flink.playground.datagen.model.TransactionSerializer; @@ -32,9 +31,6 @@ import org.apache.kafka.common.serialization.LongSerializer; /** Generates CSV transaction records at a rate */ public class Producer implements Runnable, AutoCloseable { - private static final DateTimeFormatter formatter = - DateTimeFormatter.ofPattern("yyyy-MM-dd hh:mm:ss"); - private volatile boolean isRunning; private final String brokers; diff --git a/docker/ops-playground-image/Dockerfile b/docker/ops-playground-image/Dockerfile index 74aa593..b9e5d19 100644 --- a/docker/ops-playground-image/Dockerfile +++ b/docker/ops-playground-image/Dockerfile @@ -20,7 +20,7 @@ # Build Click Count Job ############################################################################### -FROM maven:3.8.6-eclipse-temurin-17 AS builder +FROM maven:3.8-eclipse-temurin-17 AS builder # Get Click Count job and compile it COPY ./java/flink-playground-clickcountjob /opt/flink-playground-clickcountjob diff --git a/docker/ops-playground-image/java/flink-playground-clickcountjob/pom.xml b/docker/ops-playground-image/java/flink-playground-clickcountjob/pom.xml index ab08a30..f7374fa 100644 --- a/docker/ops-playground-image/java/flink-playground-clickcountjob/pom.xml +++ b/docker/ops-playground-image/java/flink-playground-clickcountjob/pom.xml @@ -169,20 +169,6 @@ under the License. <ignore/> </action> </pluginExecution> - <pluginExecution> - <pluginExecutionFilter> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <versionRange>[3.1,)</versionRange> - <goals> - <goal>testCompile</goal> - <goal>compile</goal> - </goals> - </pluginExecutionFilter> - <action> - <ignore/> - </action> - </pluginExecution> </pluginExecutions> </lifecycleMappingMetadata> </configuration> diff --git a/operations-playground/README.md b/operations-playground/README.md index 6e7cd56..e9a9405 100644 --- a/operations-playground/README.md +++ b/operations-playground/README.md @@ -61,4 +61,4 @@ docker-compose down ## Further instructions The playground setup and more detailed instructions are presented in the -["Getting Started" guide](https://ci.apache.org/projects/flink/flink-docs-release-1.15/try-flink/flink-operations-playground.html) of Flink's documentation. +["Getting Started" guide](https://ci.apache.org/projects/flink/flink-docs-release-1.16/try-flink/flink-operations-playground.html) of Flink's documentation. diff --git a/table-walkthrough/Dockerfile b/table-walkthrough/Dockerfile index d02d684..3f1347c 100644 --- a/table-walkthrough/Dockerfile +++ b/table-walkthrough/Dockerfile @@ -16,18 +16,18 @@ # limitations under the License. ################################################################################ -FROM maven:3.8-jdk-8-slim AS builder +FROM maven:3.8-eclipse-temurin-17 AS builder COPY ./pom.xml /opt/pom.xml COPY ./src /opt/src -RUN cd /opt; mvn clean install -Dmaven.test.skip +RUN cd /opt; mvn clean verify -Dmaven.test.skip -FROM apache/flink:1.15.2-scala_2.12-java8 +FROM apache/flink:1.16.0-scala_2.12-java11 # Download connector libraries -RUN wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-kafka/1.15.2/flink-sql-connector-kafka-1.15.2.jar; \ - wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-connector-jdbc/1.15.2/flink-connector-jdbc-1.15.2.jar; \ - wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-csv/1.15.2/flink-csv-1.15.2.jar; \ +RUN wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-kafka/1.16.0/flink-sql-connector-kafka-1.16.0.jar; \ + wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-connector-jdbc/1.16.0/flink-connector-jdbc-1.16.0.jar; \ + wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-csv/1.16.0/flink-csv-1.16.0.jar; \ wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.19/mysql-connector-java-8.0.19.jar; COPY --from=builder /opt/target/spend-report-*.jar /opt/flink/usrlib/spend-report.jar diff --git a/table-walkthrough/docker-compose.yml b/table-walkthrough/docker-compose.yml index 2991e45..4b55331 100644 --- a/table-walkthrough/docker-compose.yml +++ b/table-walkthrough/docker-compose.yml @@ -19,7 +19,7 @@ version: '2.1' services: jobmanager: - image: apache/flink-table-walkthrough:1-FLINK-1.15-scala_2.12 + image: apache/flink-table-walkthrough:1-FLINK-1.16-scala_2.12 build: . hostname: "jobmanager" expose: @@ -33,7 +33,7 @@ services: - kafka - mysql taskmanager: - image: apache/flink-table-walkthrough:1-FLINK-1.15-scala_2.12 + image: apache/flink-table-walkthrough:1-FLINK-1.16-scala_2.12 build: . expose: - "6121" diff --git a/table-walkthrough/pom.xml b/table-walkthrough/pom.xml index bc83e9d..82e904c 100644 --- a/table-walkthrough/pom.xml +++ b/table-walkthrough/pom.xml @@ -30,8 +30,8 @@ under the License. <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <flink.version>1.15.2</flink.version> - <java.version>1.8</java.version> + <flink.version>1.16.0</flink.version> + <java.version>11</java.version> <scala.binary.version>2.12</scala.binary.version> <maven.compiler.source>${java.version}</maven.compiler.source> <maven.compiler.target>${java.version}</maven.compiler.target> @@ -202,20 +202,6 @@ under the License. <ignore/> </action> </pluginExecution> - <pluginExecution> - <pluginExecutionFilter> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <versionRange>[3.1,)</versionRange> - <goals> - <goal>testCompile</goal> - <goal>compile</goal> - </goals> - </pluginExecutionFilter> - <action> - <ignore/> - </action> - </pluginExecution> </pluginExecutions> </lifecycleMappingMetadata> </configuration>