This is an automated email from the ASF dual-hosted git repository. pnowojski pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/flink.git
commit 9e6ff81e22d6f5f04abb50ca1aea84fd2542bf9d Author: Aleksey Pak <alek...@ververica.com> AuthorDate: Thu Jun 6 12:42:21 2019 +0200 [FLINK-12143][e2e] Change end-to-end tests with file system to use plugins mechanism --- .../test-scripts/common_dummy_fs.sh | 23 ++++++++++++++++++++++ flink-end-to-end-tests/test-scripts/common_s3.sh | 2 +- .../test-scripts/test_azure_fs.sh | 2 +- .../test-scripts/test_batch_wordcount.sh | 3 ++- .../test-scripts/test_docker_embedded_job.sh | 3 ++- .../test-scripts/test_yarn_kerberos_docker.sh | 3 ++- 6 files changed, 31 insertions(+), 5 deletions(-) diff --git a/flink-end-to-end-tests/test-scripts/common_dummy_fs.sh b/flink-end-to-end-tests/test-scripts/common_dummy_fs.sh new file mode 100644 index 0000000..bd6e756 --- /dev/null +++ b/flink-end-to-end-tests/test-scripts/common_dummy_fs.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +function dummy_fs_setup() { + mkdir -p "$FLINK_DIR/plugins/dummy-fs" + cp "${END_TO_END_DIR}/flink-plugins-test/target/flink-dummy-fs.jar" "${FLINK_DIR}/plugins/dummy-fs/" +} diff --git a/flink-end-to-end-tests/test-scripts/common_s3.sh b/flink-end-to-end-tests/test-scripts/common_s3.sh index 6536a88..b423977 100644 --- a/flink-end-to-end-tests/test-scripts/common_s3.sh +++ b/flink-end-to-end-tests/test-scripts/common_s3.sh @@ -63,7 +63,7 @@ s3util="java -jar ${END_TO_END_DIR}/flink-e2e-test-utils/target/S3UtilProgram.ja # None ################################### function s3_setup { - add_optional_lib "s3-fs-$1" + add_optional_plugin "s3-fs-$1" set_config_key "s3.access-key" "$IT_CASE_S3_ACCESS_KEY" set_config_key "s3.secret-key" "$IT_CASE_S3_SECRET_KEY" } diff --git a/flink-end-to-end-tests/test-scripts/test_azure_fs.sh b/flink-end-to-end-tests/test-scripts/test_azure_fs.sh index f68c6d8..b200e49 100755 --- a/flink-end-to-end-tests/test-scripts/test_azure_fs.sh +++ b/flink-end-to-end-tests/test-scripts/test_azure_fs.sh @@ -61,7 +61,7 @@ AZURE_TEST_DATA_WORDS_URI="wasbs://$IT_CASE_AZURE_CONTAINER@$IT_CASE_AZURE_ACCOU function azure_setup { echo "Copying flink azure jars and writing out configs" - add_optional_lib "azure-fs-hadoop" + add_optional_plugin "azure-fs-hadoop" set_config_key "fs.azure.account.key.$IT_CASE_AZURE_ACCOUNT.blob.core.windows.net" "$IT_CASE_AZURE_ACCESS_KEY" } diff --git a/flink-end-to-end-tests/test-scripts/test_batch_wordcount.sh b/flink-end-to-end-tests/test-scripts/test_batch_wordcount.sh index 3b8400b..53bd585 100755 --- a/flink-end-to-end-tests/test-scripts/test_batch_wordcount.sh +++ b/flink-end-to-end-tests/test-scripts/test_batch_wordcount.sh @@ -35,7 +35,8 @@ case $INPUT_TYPE in INPUT_LOCATION="${S3_TEST_DATA_WORDS_URI}" ;; (dummy-fs) - cp "${END_TO_END_DIR}/flink-plugins-test/target/flink-dummy-fs.jar" "${FLINK_DIR}/lib/" + source "$(dirname "$0")"/common_dummy_fs.sh + dummy_fs_setup INPUT_LOCATION="dummy://localhost/words" ;; (*) diff --git a/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh b/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh index dabb3ce..4b87d57 100755 --- a/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh +++ b/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh @@ -37,7 +37,8 @@ case $INPUT_TYPE in INPUT_LOCATION=${INPUT_PATH}/words ;; (dummy-fs) - cp "${END_TO_END_DIR}/flink-plugins-test/target/flink-dummy-fs.jar" "${FLINK_DIR}/lib/" + source "$(dirname "$0")"/common_dummy_fs.sh + dummy_fs_setup INPUT_LOCATION="dummy://localhost/words" ;; (*) diff --git a/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh b/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh index 5199b2a..6f66236 100755 --- a/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh +++ b/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh @@ -43,7 +43,8 @@ case $INPUT_TYPE in EXPECTED_RESULT_LOG_CONTAINS=("consummation,1" "of,14" "calamity,1") ;; (dummy-fs) - cp "${END_TO_END_DIR}/flink-plugins-test/target/flink-dummy-fs.jar" "${FLINK_DIR}/lib/" + source "$(dirname "$0")"/common_dummy_fs.sh + dummy_fs_setup INPUT_ARGS="--input dummy://localhost/words" EXPECTED_RESULT_LOG_CONTAINS=("my,1" "dear,2" "world,2") ;;