This is an automated email from the ASF dual-hosted git repository. fhueske pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/flink-playgrounds.git
commit eedb8b8bcb6e896251795b87f1e772b143750407 Author: Konstantin Knauf <knauf.konstan...@gmail.com> AuthorDate: Mon Jul 22 08:07:27 2019 +0200 [FLINK-12749] [playgrounds] initial version of flink-cluster-playground --- README.md | 20 ++++++ flink-cluster-playground/conf/flink-conf.yaml | 30 +++++++++ flink-cluster-playground/conf/log4j-cli.properties | 43 +++++++++++++ .../conf/log4j-console.properties | 39 ++++++++++++ flink-cluster-playground/docker-compose.yaml | 72 ++++++++++++++++++++++ 5 files changed, 204 insertions(+) diff --git a/README.md b/README.md new file mode 100644 index 0000000..8fe2903 --- /dev/null +++ b/README.md @@ -0,0 +1,20 @@ +# Apache Flink Playgrounds + +Apache Flink is an open source stream processing framework with powerful stream- and batch- +processing capabilities. + +Learn more about Flink at [http://flink.apache.org/](http://flink.apache.org/) + +## Playgrounds + +This repository contains the configuration files for two Apache Flink playgrounds. + +* The [Flink Cluster Playground](../master/flink-cluster-playground) consists of a Flink Session Cluster, a Kafka Cluster and a simple +Flink Job. It is explained in detail as part of +[Apache Flink's "Getting Started" guide](https://ci.apache.org/projects/flink/flink-docs-stable/getting-started/docker-playgrounds/flink-cluster-playground.html). + +* The interactive SQL playground is still under development and will be added shortly. + +## About + +Apache Flink is an open source project of The Apache Software Foundation (ASF). \ No newline at end of file diff --git a/flink-cluster-playground/conf/flink-conf.yaml b/flink-cluster-playground/conf/flink-conf.yaml new file mode 100644 index 0000000..5c8d0e6 --- /dev/null +++ b/flink-cluster-playground/conf/flink-conf.yaml @@ -0,0 +1,30 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +jobmanager.rpc.address: jobmanager +blob.server.port: 6124 +query.server.port: 6125 + +taskmanager.numberOfTaskSlots: 2 + +state.backend: filesystem +state.checkpoints.dir: file:///tmp/flink-checkpoints-directory +state.savepoints.dir: file:///tmp/flink-savepoints-directory + +heartbeat.interval: 1000 +heartbeat.timeout: 5000 diff --git a/flink-cluster-playground/conf/log4j-cli.properties b/flink-cluster-playground/conf/log4j-cli.properties new file mode 100644 index 0000000..a6c8d36 --- /dev/null +++ b/flink-cluster-playground/conf/log4j-cli.properties @@ -0,0 +1,43 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +log4j.rootLogger=INFO, file + +# Log all infos in the given file +log4j.appender.file=org.apache.log4j.FileAppender +log4j.appender.file.file=${log.file} +log4j.appender.file.append=false +log4j.appender.file.layout=org.apache.log4j.PatternLayout +log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + + +# Log output from org.apache.flink.yarn to the console. This is used by the +# CliFrontend class when using a per-job YARN cluster. +log4j.logger.org.apache.flink.yarn=INFO, console +log4j.logger.org.apache.flink.yarn.cli.FlinkYarnSessionCli=INFO, console +log4j.logger.org.apache.hadoop=INFO, console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + +# suppress the warning that hadoop native libraries are not loaded (irrelevant for the client) +log4j.logger.org.apache.hadoop.util.NativeCodeLoader=OFF + +# suppress the irrelevant (wrong) warnings from the netty channel handler +log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file diff --git a/flink-cluster-playground/conf/log4j-console.properties b/flink-cluster-playground/conf/log4j-console.properties new file mode 100644 index 0000000..b458d58 --- /dev/null +++ b/flink-cluster-playground/conf/log4j-console.properties @@ -0,0 +1,39 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# This affects logging for both user code and Flink +log4j.rootLogger=INFO, console + +# Uncomment this if you want to _only_ change Flink's logging +#log4j.logger.org.apache.flink=INFO + +# The following lines keep the log level of common libraries/connectors on +# log level INFO. The root logger does not override this. You have to manually +# change the log levels here. +log4j.logger.akka=INFO +log4j.logger.org.apache.kafka=INFO +log4j.logger.org.apache.hadoop=INFO +log4j.logger.org.apache.zookeeper=INFO + +# Log all infos to the console +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + +# Suppress the irrelevant (wrong) warnings from the Netty channel handler +log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, console diff --git a/flink-cluster-playground/docker-compose.yaml b/flink-cluster-playground/docker-compose.yaml new file mode 100644 index 0000000..7842762 --- /dev/null +++ b/flink-cluster-playground/docker-compose.yaml @@ -0,0 +1,72 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +version: "2.1" +services: + client: + image: flink:1.9-scala_2.11 + command: "flink run -d -p 2 /opt/flink/examples/streaming/ClickEventCount.jar --bootstrap.servers kafka:9092 --checkpointing --event-time" + depends_on: + - jobmanager + - kafka + volumes: + - ./conf:/opt/flink/conf + environment: + - JOB_MANAGER_RPC_ADDRESS=jobmanager + clickevent-generator: + image: flink:1.9-scala_2.11 + command: "java -classpath /opt/flink/examples/streaming/ClickEventCount.jar:/opt/flink/lib/* org.apache.flink.streaming.examples.windowing.clickeventcount.ClickEventGenerator --bootstrap.servers kafka:9092 --topic input" + depends_on: + - kafka + jobmanager: + image: flink:1.9-scala_2.11 + command: "jobmanager.sh start-foreground" + ports: + - 8081:8081 + volumes: + - ./conf:/opt/flink/conf + - flink-checkpoint-directory:/tmp/flink-checkpoint-directory + - /tmp/flink-savepoints-directory:/tmp/flink-savepoints-directory + environment: + - JOB_MANAGER_RPC_ADDRESS=jobmanager + taskmanager: + image: flink:1.9-scala_2.11 + depends_on: + - jobmanager + command: "taskmanager.sh start-foreground" + volumes: + - ./conf:/opt/flink/conf + - flink-checkpoint-directory:/tmp/flink-checkpoint-directory + - /tmp/flink-savepoints-directory:/tmp/flink-savepoints-directory + environment: + - JOB_MANAGER_RPC_ADDRESS=jobmanager + zookeeper: + image: wurstmeister/zookeeper:3.4.6 + kafka: + image: wurstmeister/kafka:2.12-2.2.1 + environment: + KAFKA_ADVERTISED_LISTENERS: INSIDE://:9092,OUTSIDE://:9094 + KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE + KAFKA_CREATE_TOPICS: "input:2:1, output:2:1" + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + ports: + - 9094:9094 +volumes: + flink-checkpoint-directory: \ No newline at end of file