dawidwys closed pull request #6372: [Flink-9353] Tests running per job standalone cluster with Docker & Kubernetes URL: https://github.com/apache/flink/pull/6372
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/flink-end-to-end-tests/README.md b/flink-end-to-end-tests/README.md index 26dba3d5dd3..257298e3cce 100644 --- a/flink-end-to-end-tests/README.md +++ b/flink-end-to-end-tests/README.md @@ -31,6 +31,10 @@ You can also run tests individually via $ FLINK_DIR=<flink dir> flink-end-to-end-tests/run-single-test.sh your_test.sh arg1 arg2 ``` +### Kubernetes test + +Kubernetes test (test_kubernetes_embedded_job.sh) assumes a running minikube cluster. + ## Writing Tests ### Examples diff --git a/flink-end-to-end-tests/run-pre-commit-tests.sh b/flink-end-to-end-tests/run-pre-commit-tests.sh index 6355fd0f2aa..2c963928ee1 100755 --- a/flink-end-to-end-tests/run-pre-commit-tests.sh +++ b/flink-end-to-end-tests/run-pre-commit-tests.sh @@ -51,6 +51,7 @@ run_test "Shaded Hadoop S3A end-to-end test" "$END_TO_END_DIR/test-scripts/test_ run_test "Shaded Presto S3 end-to-end test" "$END_TO_END_DIR/test-scripts/test_shaded_presto_s3.sh" run_test "Hadoop-free Wordcount end-to-end test" "$END_TO_END_DIR/test-scripts/test_hadoop_free.sh" run_test "Distributed cache end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_distributed_cache_via_blob.sh" +run_test "Wordcount end-to-end test in docker env" "$END_TO_END_DIR/test-scripts/test_docker_embedded_job.sh" printf "\n[PASS] All tests passed\n" exit 0 diff --git a/flink-end-to-end-tests/test-scripts/container-scripts/docker-compose.test.yml b/flink-end-to-end-tests/test-scripts/container-scripts/docker-compose.test.yml new file mode 100644 index 00000000000..5281111a6cb --- /dev/null +++ b/flink-end-to-end-tests/test-scripts/container-scripts/docker-compose.test.yml @@ -0,0 +1,30 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Extensions to flink-container/docker/docker-compose.yml that mounts volumes needed for tests + +version: "2.1" +services: + job-cluster: + volumes: + - ${INPUT_VOLUME}:${INPUT_PATH} + - ${OUTPUT_VOLUME}:${OUTPUT_PATH} + taskmanager: + volumes: + - ${INPUT_VOLUME}:${INPUT_PATH} + - ${OUTPUT_VOLUME}:${OUTPUT_PATH} diff --git a/flink-end-to-end-tests/test-scripts/container-scripts/job-cluster-job.yaml.template b/flink-end-to-end-tests/test-scripts/container-scripts/job-cluster-job.yaml.template new file mode 100644 index 00000000000..3785e6f11d0 --- /dev/null +++ b/flink-end-to-end-tests/test-scripts/container-scripts/job-cluster-job.yaml.template @@ -0,0 +1,46 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +apiVersion: batch/v1 +kind: Job +metadata: + name: flink-job-cluster +spec: + template: + metadata: + labels: + app: flink + component: job-cluster + spec: + restartPolicy: OnFailure + containers: + - name: flink-job-cluster + image: ${FLINK_IMAGE_NAME} + imagePullPolicy: Never + args: ["job-cluster", "--job-classname", "${FLINK_JOB}", "-Djobmanager.rpc.address=flink-job-cluster", + "-Dparallelism.default=${FLINK_JOB_PARALLELISM}", "-Dblob.server.port=6124", "-Dquery.server.ports=6125", + ${FLINK_JOB_ARGUMENTS}] + ports: + - containerPort: 6123 + name: rpc + - containerPort: 6124 + name: blob + - containerPort: 6125 + name: query + - containerPort: 8081 + name: ui diff --git a/flink-end-to-end-tests/test-scripts/container-scripts/task-manager-deployment.yaml.template b/flink-end-to-end-tests/test-scripts/container-scripts/task-manager-deployment.yaml.template new file mode 100644 index 00000000000..40d1d279444 --- /dev/null +++ b/flink-end-to-end-tests/test-scripts/container-scripts/task-manager-deployment.yaml.template @@ -0,0 +1,41 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: flink-task-manager +spec: + replicas: ${FLINK_JOB_PARALLELISM} + template: + metadata: + labels: + app: flink + component: task-manager + spec: + containers: + - name: flink-task-manager + image: ${FLINK_IMAGE_NAME} + imagePullPolicy: Never + args: ["task-manager", "-Djobmanager.rpc.address=flink-job-cluster"] + volumeMounts: + - mountPath: /cache + name: cache-volume + volumes: + - name: cache-volume + emptyDir: {} diff --git a/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh b/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh new file mode 100755 index 00000000000..370ef052a4d --- /dev/null +++ b/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +source "$(dirname "$0")"/common.sh + +DOCKER_MODULE_DIR=${END_TO_END_DIR}/../flink-container/docker +DOCKER_SCRIPTS=${END_TO_END_DIR}/test-scripts/container-scripts + +export FLINK_JOB=org.apache.flink.examples.java.wordcount.WordCount +export FLINK_DOCKER_IMAGE_NAME=test_docker_embedded_job +export INPUT_VOLUME=${END_TO_END_DIR}/test-scripts/test-data +export OUTPUT_VOLUME=${TEST_DATA_DIR}/out +export INPUT_PATH=/data/test/input +export OUTPUT_PATH=/data/test/output +export FLINK_JOB_ARGUMENTS="--input ${INPUT_PATH}/words --output ${OUTPUT_PATH}/docker_wc_out" + +# user inside the container must be able to createto workaround in-container permissions +mkdir -p $OUTPUT_VOLUME +chmod 777 $OUTPUT_VOLUME + +cd "$DOCKER_MODULE_DIR" +./build.sh --from-local-dist --job-jar ${FLINK_DIR}/examples/batch/WordCount.jar --image-name ${FLINK_DOCKER_IMAGE_NAME} +cd "$END_TO_END_DIR" + +docker-compose -f ${DOCKER_MODULE_DIR}/docker-compose.yml -f ${DOCKER_SCRIPTS}/docker-compose.test.yml up --abort-on-container-exit --exit-code-from job-cluster &> /dev/null +docker-compose -f ${DOCKER_MODULE_DIR}/docker-compose.yml -f ${DOCKER_SCRIPTS}/docker-compose.test.yml logs job-cluster > ${FLINK_DIR}/log/jobmanager.log +docker-compose -f ${DOCKER_MODULE_DIR}/docker-compose.yml -f ${DOCKER_SCRIPTS}/docker-compose.test.yml logs taskmanager > ${FLINK_DIR}/log/taskmanager.log + +check_result_hash "WordCount" $OUTPUT_VOLUME/docker_wc_out "72a690412be8928ba239c2da967328a5" diff --git a/flink-end-to-end-tests/test-scripts/test_kubernetes_embedded_job.sh b/flink-end-to-end-tests/test-scripts/test_kubernetes_embedded_job.sh new file mode 100755 index 00000000000..4529fd904a6 --- /dev/null +++ b/flink-end-to-end-tests/test-scripts/test_kubernetes_embedded_job.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +source "$(dirname "$0")"/common.sh + +DOCKER_MODULE_DIR=${END_TO_END_DIR}/../flink-container/docker +KUBERNETES_MODULE_DIR=${END_TO_END_DIR}/../flink-container/kubernetes +CONTAINER_SCRIPTS=${END_TO_END_DIR}/test-scripts/container-scripts + +export FLINK_JOB=org.apache.flink.examples.java.wordcount.WordCount +export FLINK_IMAGE_NAME=test_kubernetes_embedded_job +export OUTPUT_VOLUME=${TEST_DATA_DIR}/out +export OUTPUT_FILE=kubernetes_wc_out +export FLINK_JOB_PARALLELISM=1 +export FLINK_JOB_ARGUMENTS='"--output", "/cache/kubernetes_wc_out"' + +function cleanup { + kubectl delete job flink-job-cluster + kubectl delete service flink-job-cluster + kubectl delete deployment flink-task-manager + rm -rf ${OUTPUT_VOLUME} +} + +trap cleanup EXIT + +mkdir -p $OUTPUT_VOLUME + +eval $(minikube docker-env) +cd "$DOCKER_MODULE_DIR" +./build.sh --from-local-dist --job-jar ${FLINK_DIR}/examples/batch/WordCount.jar --image-name ${FLINK_IMAGE_NAME} +cd "$END_TO_END_DIR" + + +kubectl create -f ${KUBERNETES_MODULE_DIR}/job-cluster-service.yaml +envsubst '${FLINK_IMAGE_NAME} ${FLINK_JOB} ${FLINK_JOB_PARALLELISM} ${FLINK_JOB_ARGUMENTS}' < ${CONTAINER_SCRIPTS}/job-cluster-job.yaml.template | kubectl create -f - +envsubst '${FLINK_IMAGE_NAME} ${FLINK_JOB_PARALLELISM}' < ${CONTAINER_SCRIPTS}/task-manager-deployment.yaml.template | kubectl create -f - +kubectl wait --for=condition=complete job/flink-job-cluster --timeout=1h +kubectl cp `kubectl get pods | awk '/task-manager/ {print $1}'`:/cache/${OUTPUT_FILE} ${OUTPUT_VOLUME}/${OUTPUT_FILE} + +check_result_hash "WordCount" ${OUTPUT_VOLUME}/${OUTPUT_FILE} "e682ec6622b5e83f2eb614617d5ab2cf" ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
