This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/master by this push:
new bce3a0fc4c HOP-3946: add GCP support to integration tests
new fb348f6367 Merge pull request #1516 from hansva/master
bce3a0fc4c is described below
commit bce3a0fc4ce60229177afae4f3e2c328a0252a40
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Thu Jun 2 16:42:21 2022 +0200
HOP-3946: add GCP support to integration tests
---
.gitignore | 1 +
Jenkinsfile.daily | 4 +-
docker/integration-tests/Dockerfile.unit-tests | 5 +
.../integration-tests/integration-tests-base.yaml | 1 +
...e.hwf => disabled-main-0003-input-bigtable.hwf} | 0
integration-tests/gcp/disabled.txt | 1 -
integration-tests/scripts/run-tests-docker.sh | 107 +++++++++++++--------
integration-tests/scripts/run-tests.sh | 13 +--
8 files changed, 81 insertions(+), 51 deletions(-)
diff --git a/.gitignore b/.gitignore
index 05e1498af2..94325624d2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,6 +31,7 @@ integration-tests/surefire-reports
integration-tests/surefire-reports/spreadsheet/exelwriter-testfile.xlsx
**/*.json.old
yarn-error.log
+integration-tests/scripts/*.json
docs/content
docs/public
diff --git a/Jenkinsfile.daily b/Jenkinsfile.daily
index aaaee04918..4ec49c819d 100644
--- a/Jenkinsfile.daily
+++ b/Jenkinsfile.daily
@@ -46,6 +46,7 @@ pipeline {
jenkins_uid = sh(script: "id -u ${USER}", returnStdout: true).trim()
jenkins_gid = sh(script: "id -g ${USER}", returnStdout: true).trim()
jenkins_group = sh(script: "id -gn ${USER}", returnStdout: true).trim()
+ GCP_CREDS = credentials('gcp-access-hop')
}
options {
@@ -97,7 +98,8 @@ pipeline {
}
stage('Create Docker image & Run Tests'){
steps {
- sh "integration-tests/scripts/run-tests-docker.sh
JENKINS_USER=${jenkins_user} JENKINS_UID=${jenkins_uid}
JENKINS_GROUP=${jenkins_group} JENKINS_GID=${jenkins_gid}"
+ cp GCP_CREDS google-key-apache-hop-it.json
+ sh "integration-tests/scripts/run-tests-docker.sh
JENKINS_USER=${jenkins_user} JENKINS_UID=${jenkins_uid}
JENKINS_GROUP=${jenkins_group} JENKINS_GID=${jenkins_gid}
GCP_KEY_FILE=google-key-apache-hop-it.json"
}
post {
always {
diff --git a/docker/integration-tests/Dockerfile.unit-tests
b/docker/integration-tests/Dockerfile.unit-tests
index 49249a0017..267b18b467 100644
--- a/docker/integration-tests/Dockerfile.unit-tests
+++ b/docker/integration-tests/Dockerfile.unit-tests
@@ -30,6 +30,7 @@ ARG JENKINS_USER=hop
ARG JENKINS_GROUP=hop
ARG JENKINS_UID=1000
ARG JENKINS_GID=1000
+ARG GCP_KEY_FILE=
# Set system properties
ENV DEBIAN_FRONTEND=noninteractive
@@ -73,6 +74,9 @@ RUN pip3 install parquet-tools
COPY --chown=${JENKINS_USER}:${JENKINS_GROUP} ./assemblies/client/target/hop-*
${DEPLOYMENT_PATH}/hop.zip
+# Copy gcp key
+COPY --chown=${JENKINS_USER}:${JENKINS_GROUP} ${GCP_KEY_FILE}
/tmp/google-key-apache-hop-it.json
+
# Unzip and install in correct location
RUN unzip ${DEPLOYMENT_PATH}/hop.zip -d ${DEPLOYMENT_PATH} \
@@ -86,6 +90,7 @@ RUN unzip ${DEPLOYMENT_PATH}/hop.zip -d ${DEPLOYMENT_PATH} \
VOLUME ["/files"]
USER ${JENKINS_USER}
ENV PATH=$PATH:${DEPLOYMENT_PATH}/hop
+ENV GOOGLE_APPLICATION_CREDENTIALS="/tmp/google-key-apache-hop-it.json"
WORKDIR /home/${JENKINS_USER}
# CMD ["/bin/bash"]
ENTRYPOINT []
\ No newline at end of file
diff --git a/docker/integration-tests/integration-tests-base.yaml
b/docker/integration-tests/integration-tests-base.yaml
index 668e1710be..e716a99a00 100644
--- a/docker/integration-tests/integration-tests-base.yaml
+++ b/docker/integration-tests/integration-tests-base.yaml
@@ -26,6 +26,7 @@ services:
- JENKINS_UID=1000
- JENKINS_GROUP=jenkins
- JENKINS_GID=1000
+ - GCP_KEY_FILE=./test
volumes:
- ../../integration-tests/:/files
environment:
diff --git a/integration-tests/gcp/main-0003-input-bigtable.hwf
b/integration-tests/gcp/disabled-main-0003-input-bigtable.hwf
similarity index 100%
rename from integration-tests/gcp/main-0003-input-bigtable.hwf
rename to integration-tests/gcp/disabled-main-0003-input-bigtable.hwf
diff --git a/integration-tests/gcp/disabled.txt
b/integration-tests/gcp/disabled.txt
deleted file mode 100644
index f773025476..0000000000
--- a/integration-tests/gcp/disabled.txt
+++ /dev/null
@@ -1 +0,0 @@
-This project is disabled for now
\ No newline at end of file
diff --git a/integration-tests/scripts/run-tests-docker.sh
b/integration-tests/scripts/run-tests-docker.sh
index d6237a4b1c..07e2bc3e7b 100755
--- a/integration-tests/scripts/run-tests-docker.sh
+++ b/integration-tests/scripts/run-tests-docker.sh
@@ -18,46 +18,52 @@
#
#
-CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
DOCKER_FILES_DIR="$(cd ${CURRENT_DIR}/../../docker/integration-tests/ && pwd)"
-for ARGUMENT in "$@"
-do
+for ARGUMENT in "$@"; do
- KEY=$(echo $ARGUMENT | cut -f1 -d=)
- VALUE=$(echo $ARGUMENT | cut -f2 -d=)
-
- case "$KEY" in
- PROJECT_NAME) PROJECT_NAME=${VALUE} ;;
- JENKINS_USER) JENKINS_USER=${VALUE} ;;
- JENKINS_UID) JENKINS_UID=${VALUE} ;;
- JENKINS_GROUP) JENKINS_GROUP=${VALUE} ;;
- JENKINS_GID) JENKINS_GID=${VALUE} ;;
- *)
- esac
+ KEY=$(echo $ARGUMENT | cut -f1 -d=)
+ VALUE=$(echo $ARGUMENT | cut -f2 -d=)
+ case "$KEY" in
+ PROJECT_NAME) PROJECT_NAME=${VALUE} ;;
+ JENKINS_USER) JENKINS_USER=${VALUE} ;;
+ JENKINS_UID) JENKINS_UID=${VALUE} ;;
+ JENKINS_GROUP) JENKINS_GROUP=${VALUE} ;;
+ JENKINS_GID) JENKINS_GID=${VALUE} ;;
+ GCP_KEY_FILE) GCP_KEY_FILE=${VALUE} ;;
+ *) ;;
+ esac
done
-
if [ -z "${PROJECT_NAME}" ]; then
- PROJECT_NAME="*"
+ PROJECT_NAME="*"
fi
if [ -z "${JENKINS_USER}" ]; then
- JENKINS_USER="jenkins"
+ JENKINS_USER="jenkins"
fi
if [ -z "${JENKINS_UID}" ]; then
- JENKINS_UID="1000"
+ JENKINS_UID="1000"
fi
if [ -z "${JENKINS_GROUP}" ]; then
- JENKINS_GROUP="jenkins"
+ JENKINS_GROUP="jenkins"
fi
if [ -z "${JENKINS_GID}" ]; then
- JENKINS_GID="1000"
+ JENKINS_GID="1000"
+fi
+
+if [ -z "${SUREFIRE_REPORT}" ]; then
+ SUREFIRE_REPORT="true"
+fi
+
+if [ -z "${GCP_KEY_FILE}" ]; then
+ GCP_KEY_FILE=""
fi
#Cleanup surefire reports
@@ -65,33 +71,54 @@ rm -rf "${CURRENT_DIR}"/../surefire-reports
mkdir -p "${CURRENT_DIR}"/../surefire-reports/
#Loop over project folders
-for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/ ; do
+for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do
- if [[ "$d" != *"scripts/" ]] && [[ "$d" != *"surefire-reports/" ]] && [[
"$d" != *"hopweb/" ]]; then
+ if [[ "$d" != *"scripts/" ]] && [[ "$d" != *"surefire-reports/" ]] && [[
"$d" != *"hopweb/" ]]; then
- PROJECT_NAME=$(basename $d)
+ PROJECT_NAME=$(basename $d)
- echo "Project name: ${PROJECT_NAME}"
- echo "project path: $d"
- echo "docker compose path: ${DOCKER_FILES_DIR}"
+ echo "Project name: ${PROJECT_NAME}"
+ echo "project path: $d"
+ echo "docker compose path: ${DOCKER_FILES_DIR}"
- #Check if specific compose exists
+ #Check if specific compose exists
- if [ -f "${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml"
]; then
- echo "Project compose exists."
- PROJECT_NAME=${PROJECT_NAME} docker-compose -f
${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml build --build-arg
JENKINS_USER=${JENKINS_USER} --build-arg JENKINS_UID=${JENKINS_UID} --build-arg
JENKINS_GROUP=${JENKINS_GROUP} --build-arg JENKINS_GID=${JENKINS_GID}
- PROJECT_NAME=${PROJECT_NAME} docker-compose -f
${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml up
--abort-on-container-exit
- docker-compose -f
${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml down
- else
- echo "Project compose does not exists."
- PROJECT_NAME=${PROJECT_NAME} docker-compose -f
${DOCKER_FILES_DIR}/integration-tests-base.yaml build --build-arg
JENKINS_USER=${JENKINS_USER} --build-arg JENKINS_UID=${JENKINS_UID} --build-arg
JENKINS_GROUP=${JENKINS_GROUP} --build-arg JENKINS_GID=${JENKINS_GID}
- PROJECT_NAME=${PROJECT_NAME} docker-compose -f
${DOCKER_FILES_DIR}/integration-tests-base.yaml up --abort-on-container-exit
- docker-compose -f ${DOCKER_FILES_DIR}/integration-tests-base.yaml
down
- fi
+ if [ -f "${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml" ];
then
+ echo "Project compose exists."
+ PROJECT_NAME=${PROJECT_NAME} docker-compose -f
${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml build --build-arg
JENKINS_USER=${JENKINS_USER} --build-arg JENKINS_UID=${JENKINS_UID} --build-arg
JENKINS_GROUP=${JENKINS_GROUP} --build-arg JENKINS_GID=${JENKINS_GID}
--build-arg GCP_KEY_FILE=${GCP_KEY_FILE}
+ PROJECT_NAME=${PROJECT_NAME} docker-compose -f
${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml up
--abort-on-container-exit
+ docker-compose -f
${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml down
+ else
+ echo "Project compose does not exists."
+ PROJECT_NAME=${PROJECT_NAME} docker-compose -f
${DOCKER_FILES_DIR}/integration-tests-base.yaml build --build-arg
JENKINS_USER=${JENKINS_USER} --build-arg JENKINS_UID=${JENKINS_UID} --build-arg
JENKINS_GROUP=${JENKINS_GROUP} --build-arg JENKINS_GID=${JENKINS_GID}
--build-arg GCP_KEY_FILE=${GCP_KEY_FILE}
+ PROJECT_NAME=${PROJECT_NAME} docker-compose -f
${DOCKER_FILES_DIR}/integration-tests-base.yaml up --abort-on-container-exit
+ docker-compose -f ${DOCKER_FILES_DIR}/integration-tests-base.yaml down
+ fi
+ fi
+
+ #create final report
+ if [ "${SUREFIRE_REPORT}" == "true" ]; then
+ if [ ! -f
"${CURRENT_DIR}/../surefire-reports/surefile_${PROJECT_NAME}.xml" ]; then
+ echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
>"${CURRENT_DIR}"/../surefire-reports/surefile_${PROJECT_NAME}.xml
+ echo "<testsuite
xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\";
xsi:noNamespaceSchemaLocation=\"https://maven.apache.org/surefire/maven-surefire-plugin/xsd/surefire-test-report-3.0.xsd\";
version=\"3.0\" name=\"${PROJECT_NAME}\" time=\"0\" tests=\"0\" errors=\"0\"
skipped=\"0\" failures=\"0\">"
>>"${CURRENT_DIR}"/../surefire-reports/surefile_${PROJECT_NAME}.xml
+ echo "</testsuite>"
>>"${CURRENT_DIR}"/../surefire-reports/surefile_${PROJECT_NAME}.xml
fi
+ fi
done
#Cleanup all images
-for d in ${DOCKER_FILES_DIR}/integration-tests-*.yaml ; do
- docker-compose -f $d down --rmi all --remove-orphans
+for d in ${DOCKER_FILES_DIR}/integration-tests-*.yaml; do
+ docker-compose --log-level ERROR -f $d down --rmi all --remove-orphans
done
+
+#Print Final Results
+if [ -f "${CURRENT_DIR}/../surefire-reports/passed_tests" ]; then
+ echo -e "\033[1;32mPassed tests:"
+ PASSED_TESTS="$(cat ../surefire-reports/passed_tests)"
+ echo -e "\033[1;32m${PASSED_TESTS}"
+fi
+if [ -f "${CURRENT_DIR}/../surefire-reports/failed_tests" ]; then
+ echo -e "\033[1;91mFailed tests:"
+ FAILED_TESTS="$(cat ../surefire-reports/failed_tests)"
+ echo -e "\033[1;91m${FAILED_TESTS}"
+fi
diff --git a/integration-tests/scripts/run-tests.sh
b/integration-tests/scripts/run-tests.sh
index f3063c2ae4..3b5935a5ea 100755
--- a/integration-tests/scripts/run-tests.sh
+++ b/integration-tests/scripts/run-tests.sh
@@ -160,6 +160,8 @@ for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do
if (($exit_code >= 1)); then
errors_counter=$((errors_counter + 1))
failures_counter=$((failures_counter + 1))
+ #Write single line to overview file
+ echo $test_name >>"${CURRENT_DIR}"/../surefire-reports/failed_tests
#Create surefire xml failure
echo "<testcase name=\"$test_name\" time=\"$test_duration\">"
>>${TMP_TESTCASES}
echo "<failure type=\"$test_name\"></failure>" >>${TMP_TESTCASES}
@@ -176,6 +178,8 @@ for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do
echo "</testcase>" >>${TMP_TESTCASES}
else
+ #Write single line to overview file
+ echo $test_name >>"${CURRENT_DIR}"/../surefire-reports/passed_tests
#Create surefire xml success
echo "<testcase name=\"$test_name\" time=\"$test_duration\">"
>>${TMP_TESTCASES}
echo "<system-out>" >>${TMP_TESTCASES}
@@ -197,15 +201,6 @@ for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do
total_duration=$((SECONDS - start_time))
- #Print End results
- echo ${SPACER}
- echo "Final Report"
- echo ${SPACER}
- echo "Number of Tests: $test_counter"
- echo "Total errors: $errors_counter"
- echo "Total failures: $failures_counter"
- echo "Total duration: $total_duration"
-
#create final report
if [ "${SUREFIRE_REPORT}" == "true" ]; then
