This is an automated email from the ASF dual-hosted git repository. martijnvisser pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/flink-statefun-docker.git
commit 01b731800e3f6a22822b89833c1f5e94e80cfcd4 Author: Martijn Visser <mvis...@confluent.io> AuthorDate: Fri Sep 8 13:50:47 2023 +0200 [release] Update Dockerfiles for 3.3.0 release --- 3.3.0-java11/Dockerfile | 84 +++++++++++++++ 3.3.0-java11/docker-entry-point.sh | 56 ++++++++++ .../flink-distribution/bin/flink-console.sh | 113 +++++++++++++++++++++ .../flink-distribution/bin/standalone-job.sh | 54 ++++++++++ .../flink-distribution/conf/flink-conf.yaml | 63 ++++++++++++ 3.3.0-java8/Dockerfile | 84 +++++++++++++++ 3.3.0-java8/docker-entry-point.sh | 56 ++++++++++ .../flink-distribution/bin/flink-console.sh | 113 +++++++++++++++++++++ .../flink-distribution/bin/standalone-job.sh | 54 ++++++++++ .../flink-distribution/conf/flink-conf.yaml | 63 ++++++++++++ 10 files changed, 740 insertions(+) diff --git a/3.3.0-java11/Dockerfile b/3.3.0-java11/Dockerfile new file mode 100644 index 0000000..c171642 --- /dev/null +++ b/3.3.0-java11/Dockerfile @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM flink:1.16.2-scala_2.12-java11 + +ENV STATEFUN_VERSION=3.3.0 \ + GPG_KEY=A5F3BCE4CBE993573EC5966A65321B8382B219AF + +ENV ROLE worker +ENV MASTER_HOST localhost +ENV STATEFUN_HOME /opt/statefun +ENV STATEFUN_MODULES $STATEFUN_HOME/modules + +# Cleanup flink-lib +RUN rm -fr $FLINK_HOME/lib/flink-table*jar + +# Copy our distriubtion template +COPY flink-distribution/ $FLINK_HOME/ + +# Install Stateful Functions dependencies in Flink lib +ENV DIST_JAR_URL=https://repo.maven.apache.org/maven2/org/apache/flink/statefun-flink-distribution/${STATEFUN_VERSION}/statefun-flink-distribution-${STATEFUN_VERSION}.jar \ + DIST_ASC_URL=https://repo.maven.apache.org/maven2/org/apache/flink/statefun-flink-distribution/${STATEFUN_VERSION}/statefun-flink-distribution-${STATEFUN_VERSION}.jar.asc \ + CORE_JAR_URL=https://repo.maven.apache.org/maven2/org/apache/flink/statefun-flink-core/${STATEFUN_VERSION}/statefun-flink-core-${STATEFUN_VERSION}.jar \ + CORE_ASC_URL=https://repo.maven.apache.org/maven2/org/apache/flink/statefun-flink-core/${STATEFUN_VERSION}/statefun-flink-core-${STATEFUN_VERSION}.jar.asc + +RUN set -ex; \ + wget -nv -O statefun-flink-distribution.jar "$DIST_JAR_URL"; \ + wget -nv -O statefun-flink-distribution.jar.asc "$DIST_ASC_URL"; \ + wget -nv -O statefun-flink-core.jar "$CORE_JAR_URL"; \ + wget -nv -O statefun-flink-core.jar.asc "$CORE_ASC_URL"; \ + \ + export GNUPGHOME="$(mktemp -d)"; \ + for server in ha.pool.sks-keyservers.net $(shuf -e \ + hkp://p80.pool.sks-keyservers.net:80 \ + keyserver.ubuntu.com \ + hkp://keyserver.ubuntu.com:80 \ + pgp.mit.edu) ; do \ + gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \ + done && \ + gpg --batch --verify statefun-flink-distribution.jar.asc statefun-flink-distribution.jar; \ + gpg --batch --verify statefun-flink-core.jar.asc statefun-flink-core.jar; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" statefun-flink-distribution.jar.asc statefun-flink-core.jar.asc; \ + \ + mkdir -p $FLINK_HOME/lib; \ + mv statefun-flink-distribution.jar $FLINK_HOME/lib; \ + mv statefun-flink-core.jar $FLINK_HOME/lib; + +# add user modules +USER root + +RUN mkdir -p $STATEFUN_MODULES && \ + useradd --system --home-dir $STATEFUN_HOME --uid=9998 --gid=flink statefun && \ + chown -R statefun:flink $STATEFUN_HOME && \ + chmod -R g+rw $STATEFUN_HOME + +# add filesystem plugins +RUN mkdir -p $FLINK_HOME/plugins/s3-fs-presto && \ + mv $FLINK_HOME/opt/flink-s3-fs-presto-1.16.2.jar $FLINK_HOME/plugins/s3-fs-presto +RUN mkdir -p $FLINK_HOME/plugins/oss-fs-hadoop && \ + mv $FLINK_HOME/opt/flink-oss-fs-hadoop-1.16.2.jar $FLINK_HOME/plugins/oss-fs-hadoop +RUN mkdir -p $FLINK_HOME/plugins/azure-fs-hadoop && \ + mv $FLINK_HOME/opt/flink-azure-fs-hadoop-1.16.2.jar $FLINK_HOME/plugins/azure-fs-hadoop + + +# add tcnative +RUN mv $FLINK_HOME/opt/flink-shaded-netty-tcnative-dynamic-*.jar $FLINK_HOME/lib/ + +# entry point +ADD docker-entry-point.sh /docker-entry-point.sh + +ENTRYPOINT ["/docker-entry-point.sh"] diff --git a/3.3.0-java11/docker-entry-point.sh b/3.3.0-java11/docker-entry-point.sh new file mode 100755 index 0000000..4e502b4 --- /dev/null +++ b/3.3.0-java11/docker-entry-point.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# +# Role types +# +WORKER="worker" +MASTER="master" + +# +# Environment +# +FLINK_HOME=${FLINK_HOME:-"/opt/flink/bin"} +ROLE=${ROLE:-"worker"} +MASTER_HOST=${MASTER_HOST:-"localhost"} + +# +# Start a service depending on the role. +# +if [[ "${ROLE}" == "${WORKER}" ]]; then + # + # start the TaskManager (worker role) + # + exec ${FLINK_HOME}/bin/taskmanager.sh start-foreground \ + -Djobmanager.rpc.address=${MASTER_HOST} + +elif [[ "${ROLE}" == "${MASTER}" ]]; then + # + # start the JobManager (master role) with our predefined job. + # + exec $FLINK_HOME/bin/standalone-job.sh \ + start-foreground \ + -Djobmanager.rpc.address=${MASTER_HOST} \ + "$@" +else + # + # unknown role + # + echo "unknown role ${ROLE}" + exit 1 +fi diff --git a/3.3.0-java11/flink-distribution/bin/flink-console.sh b/3.3.0-java11/flink-distribution/bin/flink-console.sh new file mode 100755 index 0000000..8fc0b05 --- /dev/null +++ b/3.3.0-java11/flink-distribution/bin/flink-console.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This file was taken from Apache Flink, and modified to include another entry point + +# Start a Flink service as a console application. Must be stopped with Ctrl-C +# or with SIGTERM by kill or the controlling process. +USAGE="Usage: flink-console.sh (taskexecutor|zookeeper|historyserver|standalonesession|standalonejob|statefun) [args]" + +SERVICE=$1 +ARGS=("${@:2}") # get remaining arguments as array + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/config.sh + +case $SERVICE in + (taskexecutor) + CLASS_TO_RUN=org.apache.flink.runtime.taskexecutor.TaskManagerRunner + ;; + + (historyserver) + CLASS_TO_RUN=org.apache.flink.runtime.webmonitor.history.HistoryServer + ;; + + (zookeeper) + CLASS_TO_RUN=org.apache.flink.runtime.zookeeper.FlinkZooKeeperQuorumPeer + ;; + + (standalonesession) + CLASS_TO_RUN=org.apache.flink.runtime.entrypoint.StandaloneSessionClusterEntrypoint + ;; + + (standalonejob) + CLASS_TO_RUN=org.apache.flink.container.entrypoint.StandaloneApplicationClusterEntryPoint + ;; + + (statefun) + CLASS_TO_RUN=org.apache.flink.statefun.flink.launcher.StatefulFunctionsClusterEntryPoint + ;; + + (*) + echo "Unknown service '${SERVICE}'. $USAGE." + exit 1 + ;; +esac + +FLINK_TM_CLASSPATH=`constructFlinkClassPath` + +if [ "$FLINK_IDENT_STRING" = "" ]; then + FLINK_IDENT_STRING="$USER" +fi + +pid=$FLINK_PID_DIR/flink-$FLINK_IDENT_STRING-$SERVICE.pid +mkdir -p "$FLINK_PID_DIR" +# The lock needs to be released after use because this script is started foreground +command -v flock >/dev/null 2>&1 +flock_exist=$? +if [[ ${flock_exist} -eq 0 ]]; then + exec 200<"$FLINK_PID_DIR" + flock 200 +fi +# Remove the pid file when all the processes are dead +if [ -f "$pid" ]; then + all_dead=0 + while read each_pid; do + # Check whether the process is still running + kill -0 $each_pid > /dev/null 2>&1 + [[ $? -eq 0 ]] && all_dead=1 + done < "$pid" + [ ${all_dead} -eq 0 ] && rm $pid +fi +id=$([ -f "$pid" ] && echo $(wc -l < "$pid") || echo "0") + +FLINK_LOG_PREFIX="${FLINK_LOG_DIR}/flink-${FLINK_IDENT_STRING}-${SERVICE}-${id}-${HOSTNAME}" +log="${FLINK_LOG_PREFIX}.log" + +log_setting=("-Dlog.file=${log}" "-Dlog4j.configuration=file:${FLINK_CONF_DIR}/log4j-console.properties" "-Dlog4j.configurationFile=file:${FLINK_CONF_DIR}/log4j-console.properties" "-Dlogback.configurationFile=file:${FLINK_CONF_DIR}/logback-console.xml") + +JAVA_VERSION=$(${JAVA_RUN} -version 2>&1 | sed 's/.*version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q') + +# Only set JVM 8 arguments if we have correctly extracted the version +if [[ ${JAVA_VERSION} =~ ${IS_NUMBER} ]]; then + if [ "$JAVA_VERSION" -lt 18 ]; then + JVM_ARGS="$JVM_ARGS -XX:MaxPermSize=256m" + fi +fi + +echo "Starting $SERVICE as a console application on host $HOSTNAME." + +# Add the current process id to pid file +echo $$ >> "$pid" 2>/dev/null + +# Release the lock because the java process runs in the foreground and would block other processes from modifying the pid file +[[ ${flock_exist} -eq 0 ]] && flock -u 200 + +exec $JAVA_RUN $JVM_ARGS ${FLINK_ENV_JAVA_OPTS} "${log_setting[@]}" -classpath "`manglePathList "$FLINK_TM_CLASSPATH:$INTERNAL_HADOOP_CLASSPATHS"`" ${CLASS_TO_RUN} "${ARGS[@]}" diff --git a/3.3.0-java11/flink-distribution/bin/standalone-job.sh b/3.3.0-java11/flink-distribution/bin/standalone-job.sh new file mode 100755 index 0000000..6e02d6c --- /dev/null +++ b/3.3.0-java11/flink-distribution/bin/standalone-job.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This file was taken from Apache Flink, and modified to include another entry point + +# Start/stop a Flink JobManager. +USAGE="Usage: standalone-job.sh ((start|start-foreground))|stop [args]" + +STARTSTOP=$1 +ENTRY_POINT_NAME="statefun" + +if [[ $STARTSTOP != "start" ]] && [[ $STARTSTOP != "start-foreground" ]] && [[ $STARTSTOP != "stop" ]]; then + echo $USAGE + exit 1 +fi + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/config.sh + +# Startup parameters +ARGS=("--configDir" "${FLINK_CONF_DIR}" "${@:2}") + +if [[ $STARTSTOP == "start" ]] || [[ $STARTSTOP == "start-foreground" ]]; then + # Add cluster entry point specific JVM options + export FLINK_ENV_JAVA_OPTS="${FLINK_ENV_JAVA_OPTS} ${FLINK_ENV_JAVA_OPTS_JM}" + parseJmArgsAndExportLogs "${ARGS[@]}" + + if [ ! -z "${DYNAMIC_PARAMETERS}" ]; then + ARGS+=(${DYNAMIC_PARAMETERS[@]}) + fi +fi + +if [[ $STARTSTOP == "start-foreground" ]]; then + exec "${FLINK_BIN_DIR}"/flink-console.sh ${ENTRY_POINT_NAME} "${ARGS[@]}" +else + "${FLINK_BIN_DIR}"/flink-daemon.sh ${STARTSTOP} ${ENTRY_POINT_NAME} "${ARGS[@]}" +fi diff --git a/3.3.0-java11/flink-distribution/conf/flink-conf.yaml b/3.3.0-java11/flink-distribution/conf/flink-conf.yaml new file mode 100644 index 0000000..e2c7b13 --- /dev/null +++ b/3.3.0-java11/flink-distribution/conf/flink-conf.yaml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# This file is the base for the Apache Flink configuration + +statefun.flink-job-name: Statefun Application + +#============================================================================== +# Configurations strictly required by Stateful Functions. Do not change. +#============================================================================== + +classloader.parent-first-patterns.additional: org.apache.flink.statefun;org.apache.kafka;com.google.protobuf + +#============================================================================== +# Fault tolerance, checkpointing and recovery. +# For more related configuration options, please see: https://ci.apache.org/projects/flink/flink-docs-master/ops/config.html#fault-tolerance +#============================================================================== + +# Uncomment the below to enable checkpointing for your application +#execution.checkpointing.mode: EXACTLY_ONCE +#execution.checkpointing.interval: 5sec + +restart-strategy: fixed-delay +restart-strategy.fixed-delay.attempts: 2147483647 +restart-strategy.fixed-delay.delay: 1sec + +state.backend.local-recovery: true +state.backend: rocksdb +state.backend.rocksdb.timer-service.factory: ROCKSDB +state.backend.rocksdb.localdir: /local/state/rocksdb +state.backend.rocksdb.memory.partitioned-index-filters: true +state.backend.rocksdb.checkpoint.transfer.thread.num: 8 +state.backend.rocksdb.thread.num: 4 +state.checkpoints.dir: file:///checkpoint-dir +state.backend.incremental: true + +taskmanager.state.local.root-dirs: file:///local/state/recovery + +#============================================================================== +# Recommended memory configurations. Users may change according to their needs. +#============================================================================== + +jobmanager.memory.process.size: 1g +taskmanager.memory.process.size: 4g + +#============================================================================== +# Support easy upgrades as the module.yaml file updates +#============================================================================== + +pipeline.auto-generate-uids: false +execution.savepoint.ignore-unclaimed-state: true + diff --git a/3.3.0-java8/Dockerfile b/3.3.0-java8/Dockerfile new file mode 100644 index 0000000..69b70e6 --- /dev/null +++ b/3.3.0-java8/Dockerfile @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM flink:1.16.2-scala_2.12-java8 + +ENV STATEFUN_VERSION=3.3.0 \ + GPG_KEY=A5F3BCE4CBE993573EC5966A65321B8382B219AF + +ENV ROLE worker +ENV MASTER_HOST localhost +ENV STATEFUN_HOME /opt/statefun +ENV STATEFUN_MODULES $STATEFUN_HOME/modules + +# Cleanup flink-lib +RUN rm -fr $FLINK_HOME/lib/flink-table*jar + +# Copy our distriubtion template +COPY flink-distribution/ $FLINK_HOME/ + +# Install Stateful Functions dependencies in Flink lib +ENV DIST_JAR_URL=https://repo.maven.apache.org/maven2/org/apache/flink/statefun-flink-distribution/${STATEFUN_VERSION}/statefun-flink-distribution-${STATEFUN_VERSION}.jar \ + DIST_ASC_URL=https://repo.maven.apache.org/maven2/org/apache/flink/statefun-flink-distribution/${STATEFUN_VERSION}/statefun-flink-distribution-${STATEFUN_VERSION}.jar.asc \ + CORE_JAR_URL=https://repo.maven.apache.org/maven2/org/apache/flink/statefun-flink-core/${STATEFUN_VERSION}/statefun-flink-core-${STATEFUN_VERSION}.jar \ + CORE_ASC_URL=https://repo.maven.apache.org/maven2/org/apache/flink/statefun-flink-core/${STATEFUN_VERSION}/statefun-flink-core-${STATEFUN_VERSION}.jar.asc + +RUN set -ex; \ + wget -nv -O statefun-flink-distribution.jar "$DIST_JAR_URL"; \ + wget -nv -O statefun-flink-distribution.jar.asc "$DIST_ASC_URL"; \ + wget -nv -O statefun-flink-core.jar "$CORE_JAR_URL"; \ + wget -nv -O statefun-flink-core.jar.asc "$CORE_ASC_URL"; \ + \ + export GNUPGHOME="$(mktemp -d)"; \ + for server in ha.pool.sks-keyservers.net $(shuf -e \ + hkp://p80.pool.sks-keyservers.net:80 \ + keyserver.ubuntu.com \ + hkp://keyserver.ubuntu.com:80 \ + pgp.mit.edu) ; do \ + gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \ + done && \ + gpg --batch --verify statefun-flink-distribution.jar.asc statefun-flink-distribution.jar; \ + gpg --batch --verify statefun-flink-core.jar.asc statefun-flink-core.jar; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" statefun-flink-distribution.jar.asc statefun-flink-core.jar.asc; \ + \ + mkdir -p $FLINK_HOME/lib; \ + mv statefun-flink-distribution.jar $FLINK_HOME/lib; \ + mv statefun-flink-core.jar $FLINK_HOME/lib; + +# add user modules +USER root + +RUN mkdir -p $STATEFUN_MODULES && \ + useradd --system --home-dir $STATEFUN_HOME --uid=9998 --gid=flink statefun && \ + chown -R statefun:flink $STATEFUN_HOME && \ + chmod -R g+rw $STATEFUN_HOME + +# add filesystem plugins +RUN mkdir -p $FLINK_HOME/plugins/s3-fs-presto && \ + mv $FLINK_HOME/opt/flink-s3-fs-presto-1.16.2.jar $FLINK_HOME/plugins/s3-fs-presto +RUN mkdir -p $FLINK_HOME/plugins/oss-fs-hadoop && \ + mv $FLINK_HOME/opt/flink-oss-fs-hadoop-1.16.2.jar $FLINK_HOME/plugins/oss-fs-hadoop +RUN mkdir -p $FLINK_HOME/plugins/azure-fs-hadoop && \ + mv $FLINK_HOME/opt/flink-azure-fs-hadoop-1.16.2.jar $FLINK_HOME/plugins/azure-fs-hadoop + + +# add tcnative +RUN mv $FLINK_HOME/opt/flink-shaded-netty-tcnative-dynamic-*.jar $FLINK_HOME/lib/ + +# entry point +ADD docker-entry-point.sh /docker-entry-point.sh + +ENTRYPOINT ["/docker-entry-point.sh"] diff --git a/3.3.0-java8/docker-entry-point.sh b/3.3.0-java8/docker-entry-point.sh new file mode 100755 index 0000000..4e502b4 --- /dev/null +++ b/3.3.0-java8/docker-entry-point.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# +# Role types +# +WORKER="worker" +MASTER="master" + +# +# Environment +# +FLINK_HOME=${FLINK_HOME:-"/opt/flink/bin"} +ROLE=${ROLE:-"worker"} +MASTER_HOST=${MASTER_HOST:-"localhost"} + +# +# Start a service depending on the role. +# +if [[ "${ROLE}" == "${WORKER}" ]]; then + # + # start the TaskManager (worker role) + # + exec ${FLINK_HOME}/bin/taskmanager.sh start-foreground \ + -Djobmanager.rpc.address=${MASTER_HOST} + +elif [[ "${ROLE}" == "${MASTER}" ]]; then + # + # start the JobManager (master role) with our predefined job. + # + exec $FLINK_HOME/bin/standalone-job.sh \ + start-foreground \ + -Djobmanager.rpc.address=${MASTER_HOST} \ + "$@" +else + # + # unknown role + # + echo "unknown role ${ROLE}" + exit 1 +fi diff --git a/3.3.0-java8/flink-distribution/bin/flink-console.sh b/3.3.0-java8/flink-distribution/bin/flink-console.sh new file mode 100755 index 0000000..8fc0b05 --- /dev/null +++ b/3.3.0-java8/flink-distribution/bin/flink-console.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This file was taken from Apache Flink, and modified to include another entry point + +# Start a Flink service as a console application. Must be stopped with Ctrl-C +# or with SIGTERM by kill or the controlling process. +USAGE="Usage: flink-console.sh (taskexecutor|zookeeper|historyserver|standalonesession|standalonejob|statefun) [args]" + +SERVICE=$1 +ARGS=("${@:2}") # get remaining arguments as array + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/config.sh + +case $SERVICE in + (taskexecutor) + CLASS_TO_RUN=org.apache.flink.runtime.taskexecutor.TaskManagerRunner + ;; + + (historyserver) + CLASS_TO_RUN=org.apache.flink.runtime.webmonitor.history.HistoryServer + ;; + + (zookeeper) + CLASS_TO_RUN=org.apache.flink.runtime.zookeeper.FlinkZooKeeperQuorumPeer + ;; + + (standalonesession) + CLASS_TO_RUN=org.apache.flink.runtime.entrypoint.StandaloneSessionClusterEntrypoint + ;; + + (standalonejob) + CLASS_TO_RUN=org.apache.flink.container.entrypoint.StandaloneApplicationClusterEntryPoint + ;; + + (statefun) + CLASS_TO_RUN=org.apache.flink.statefun.flink.launcher.StatefulFunctionsClusterEntryPoint + ;; + + (*) + echo "Unknown service '${SERVICE}'. $USAGE." + exit 1 + ;; +esac + +FLINK_TM_CLASSPATH=`constructFlinkClassPath` + +if [ "$FLINK_IDENT_STRING" = "" ]; then + FLINK_IDENT_STRING="$USER" +fi + +pid=$FLINK_PID_DIR/flink-$FLINK_IDENT_STRING-$SERVICE.pid +mkdir -p "$FLINK_PID_DIR" +# The lock needs to be released after use because this script is started foreground +command -v flock >/dev/null 2>&1 +flock_exist=$? +if [[ ${flock_exist} -eq 0 ]]; then + exec 200<"$FLINK_PID_DIR" + flock 200 +fi +# Remove the pid file when all the processes are dead +if [ -f "$pid" ]; then + all_dead=0 + while read each_pid; do + # Check whether the process is still running + kill -0 $each_pid > /dev/null 2>&1 + [[ $? -eq 0 ]] && all_dead=1 + done < "$pid" + [ ${all_dead} -eq 0 ] && rm $pid +fi +id=$([ -f "$pid" ] && echo $(wc -l < "$pid") || echo "0") + +FLINK_LOG_PREFIX="${FLINK_LOG_DIR}/flink-${FLINK_IDENT_STRING}-${SERVICE}-${id}-${HOSTNAME}" +log="${FLINK_LOG_PREFIX}.log" + +log_setting=("-Dlog.file=${log}" "-Dlog4j.configuration=file:${FLINK_CONF_DIR}/log4j-console.properties" "-Dlog4j.configurationFile=file:${FLINK_CONF_DIR}/log4j-console.properties" "-Dlogback.configurationFile=file:${FLINK_CONF_DIR}/logback-console.xml") + +JAVA_VERSION=$(${JAVA_RUN} -version 2>&1 | sed 's/.*version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q') + +# Only set JVM 8 arguments if we have correctly extracted the version +if [[ ${JAVA_VERSION} =~ ${IS_NUMBER} ]]; then + if [ "$JAVA_VERSION" -lt 18 ]; then + JVM_ARGS="$JVM_ARGS -XX:MaxPermSize=256m" + fi +fi + +echo "Starting $SERVICE as a console application on host $HOSTNAME." + +# Add the current process id to pid file +echo $$ >> "$pid" 2>/dev/null + +# Release the lock because the java process runs in the foreground and would block other processes from modifying the pid file +[[ ${flock_exist} -eq 0 ]] && flock -u 200 + +exec $JAVA_RUN $JVM_ARGS ${FLINK_ENV_JAVA_OPTS} "${log_setting[@]}" -classpath "`manglePathList "$FLINK_TM_CLASSPATH:$INTERNAL_HADOOP_CLASSPATHS"`" ${CLASS_TO_RUN} "${ARGS[@]}" diff --git a/3.3.0-java8/flink-distribution/bin/standalone-job.sh b/3.3.0-java8/flink-distribution/bin/standalone-job.sh new file mode 100755 index 0000000..6e02d6c --- /dev/null +++ b/3.3.0-java8/flink-distribution/bin/standalone-job.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This file was taken from Apache Flink, and modified to include another entry point + +# Start/stop a Flink JobManager. +USAGE="Usage: standalone-job.sh ((start|start-foreground))|stop [args]" + +STARTSTOP=$1 +ENTRY_POINT_NAME="statefun" + +if [[ $STARTSTOP != "start" ]] && [[ $STARTSTOP != "start-foreground" ]] && [[ $STARTSTOP != "stop" ]]; then + echo $USAGE + exit 1 +fi + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/config.sh + +# Startup parameters +ARGS=("--configDir" "${FLINK_CONF_DIR}" "${@:2}") + +if [[ $STARTSTOP == "start" ]] || [[ $STARTSTOP == "start-foreground" ]]; then + # Add cluster entry point specific JVM options + export FLINK_ENV_JAVA_OPTS="${FLINK_ENV_JAVA_OPTS} ${FLINK_ENV_JAVA_OPTS_JM}" + parseJmArgsAndExportLogs "${ARGS[@]}" + + if [ ! -z "${DYNAMIC_PARAMETERS}" ]; then + ARGS+=(${DYNAMIC_PARAMETERS[@]}) + fi +fi + +if [[ $STARTSTOP == "start-foreground" ]]; then + exec "${FLINK_BIN_DIR}"/flink-console.sh ${ENTRY_POINT_NAME} "${ARGS[@]}" +else + "${FLINK_BIN_DIR}"/flink-daemon.sh ${STARTSTOP} ${ENTRY_POINT_NAME} "${ARGS[@]}" +fi diff --git a/3.3.0-java8/flink-distribution/conf/flink-conf.yaml b/3.3.0-java8/flink-distribution/conf/flink-conf.yaml new file mode 100644 index 0000000..e2c7b13 --- /dev/null +++ b/3.3.0-java8/flink-distribution/conf/flink-conf.yaml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# This file is the base for the Apache Flink configuration + +statefun.flink-job-name: Statefun Application + +#============================================================================== +# Configurations strictly required by Stateful Functions. Do not change. +#============================================================================== + +classloader.parent-first-patterns.additional: org.apache.flink.statefun;org.apache.kafka;com.google.protobuf + +#============================================================================== +# Fault tolerance, checkpointing and recovery. +# For more related configuration options, please see: https://ci.apache.org/projects/flink/flink-docs-master/ops/config.html#fault-tolerance +#============================================================================== + +# Uncomment the below to enable checkpointing for your application +#execution.checkpointing.mode: EXACTLY_ONCE +#execution.checkpointing.interval: 5sec + +restart-strategy: fixed-delay +restart-strategy.fixed-delay.attempts: 2147483647 +restart-strategy.fixed-delay.delay: 1sec + +state.backend.local-recovery: true +state.backend: rocksdb +state.backend.rocksdb.timer-service.factory: ROCKSDB +state.backend.rocksdb.localdir: /local/state/rocksdb +state.backend.rocksdb.memory.partitioned-index-filters: true +state.backend.rocksdb.checkpoint.transfer.thread.num: 8 +state.backend.rocksdb.thread.num: 4 +state.checkpoints.dir: file:///checkpoint-dir +state.backend.incremental: true + +taskmanager.state.local.root-dirs: file:///local/state/recovery + +#============================================================================== +# Recommended memory configurations. Users may change according to their needs. +#============================================================================== + +jobmanager.memory.process.size: 1g +taskmanager.memory.process.size: 4g + +#============================================================================== +# Support easy upgrades as the module.yaml file updates +#============================================================================== + +pipeline.auto-generate-uids: false +execution.savepoint.ignore-unclaimed-state: true +