This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new a091fb1618 docker and jenkins test cleanup, #2548 (#3875)
a091fb1618 is described below
commit a091fb1618a6b9da1327471c7e617b45089ec2d7
Author: Hans Van Akelyen <hans.van.akel...@gmail.com>
AuthorDate: Fri Apr 26 16:20:29 2024 +0200
docker and jenkins test cleanup, #2548 (#3875)
---
Jenkinsfile.daily | 22 +++++------
docker/integration-tests/Dockerfile.unit-tests | 16 ++++----
.../integration-tests/integration-tests-azure.yaml | 1 -
.../integration-tests-base-spark.yaml | 1 -
.../integration-tests/integration-tests-base.yaml | 1 -
.../integration-tests-cassandra.yaml | 1 -
.../integration-tests-database.yaml | 1 -
.../integration-tests/integration-tests-flink.yaml | 1 -
.../integration-tests-hop_server.yaml | 1 -
.../integration-tests/integration-tests-http.yaml | 1 -
.../integration-tests/integration-tests-kafka.yaml | 1 -
.../integration-tests/integration-tests-mail.yaml | 1 -
.../integration-tests-monetdb.yaml | 1 -
.../integration-tests/integration-tests-mongo.yaml | 1 -
.../integration-tests/integration-tests-neo4j.yaml | 1 -
.../integration-tests/integration-tests-snmp.yaml | 1 -
.../integration-tests/integration-tests-spark.yaml | 1 -
.../integration-tests-vertica.yaml | 1 -
.../integration-tests/integration-tests-xml.yaml | 1 -
integration-tests/scripts/run-tests-docker.sh | 46 ++++++++++++----------
20 files changed, 44 insertions(+), 57 deletions(-)
diff --git a/Jenkinsfile.daily b/Jenkinsfile.daily
index 20f1b7179a..15ed9cc762 100644
--- a/Jenkinsfile.daily
+++ b/Jenkinsfile.daily
@@ -85,17 +85,17 @@ pipeline {
sh "mvn $MAVEN_PARAMS clean install"
}
}
- stage('Code Quality') {
- tools{
- jdk 'jdk_17_latest'
- }
- steps {
- echo 'Checking Code Quality on SonarCloud'
- withCredentials([string(credentialsId:
'sonarcloud-key-apache-hop', variable: 'SONAR_TOKEN')]) {
- sh 'mvn sonar:sonar -Dsonar.host.url=https://sonarcloud.io
-Dsonar.organization=apache -Dsonar.projectKey=apache_hop
-Dsonar.branch.name=${BRANCH_NAME} -Dsonar.login=${SONAR_TOKEN}'
- }
- }
- }
+ // stage('Code Quality') {
+ // tools{
+ // jdk 'jdk_17_latest'
+ // }
+ // steps {
+ // echo 'Checking Code Quality on SonarCloud'
+ // withCredentials([string(credentialsId:
'sonarcloud-key-apache-hop', variable: 'SONAR_TOKEN')]) {
+ // sh 'mvn sonar:sonar
-Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache
-Dsonar.projectKey=apache_hop -Dsonar.branch.name=${BRANCH_NAME}
-Dsonar.login=${SONAR_TOKEN}'
+ // }
+ // }
+ // }
stage('Create Docker image & Run Tests'){
steps {
sh 'cp $GCP_CREDS google-key-apache-hop-it.json'
diff --git a/docker/integration-tests/Dockerfile.unit-tests
b/docker/integration-tests/Dockerfile.unit-tests
index 8820545133..e1a652864e 100644
--- a/docker/integration-tests/Dockerfile.unit-tests
+++ b/docker/integration-tests/Dockerfile.unit-tests
@@ -16,7 +16,7 @@
#
FROM ubuntu:22.04
-MAINTAINER Apache Hop
+LABEL maintainer="Apache Hop Team"
# Argument Branch name, used to download correct version
ARG BRANCH_NAME
@@ -53,6 +53,7 @@ RUN apt-get update \
unzip \
ttf-mscorefonts-installer \
locales \
+ && apt-get clean \
&& mkdir ${VOLUME_MOUNT_POINT} \
&& addgroup -gid ${JENKINS_GID} ${JENKINS_GROUP} \
&& useradd -m -d /home/${JENKINS_USER} -u ${JENKINS_UID} -g
${JENKINS_GROUP} ${JENKINS_USER} \
@@ -72,22 +73,19 @@ RUN pip3 install parquet-tools
# Copy the hop package from the local resources folder to the container image
directory
-COPY --chown=${JENKINS_USER}:${JENKINS_GROUP} ./assemblies/client/target/hop-*
${DEPLOYMENT_PATH}/hop.zip
+COPY --chown=${JENKINS_USER}:${JENKINS_GROUP} ./assemblies/client/target/hop
${DEPLOYMENT_PATH}/hop
# Copy gcp key
COPY --chown=${JENKINS_USER}:${JENKINS_GROUP} ${GCP_KEY_FILE}
/tmp/google-key-apache-hop-it.json
# Unzip and install in correct location
-RUN unzip -q ${DEPLOYMENT_PATH}/hop.zip -d ${DEPLOYMENT_PATH} \
- && rm ${DEPLOYMENT_PATH}/hop.zip \
- && chown -R ${JENKINS_USER}:${JENKINS_GROUP} ${DEPLOYMENT_PATH}/hop \
+RUN chown -R ${JENKINS_USER}:${JENKINS_GROUP} ${DEPLOYMENT_PATH}/hop \
&& chmod 700 ${DEPLOYMENT_PATH}/hop/*.sh \
&& cd ${DEPLOYMENT_PATH}/hop \
- && ./hop-conf.sh --generate-fat-jar=/tmp/hop-fatjar.jar
-
-# Download the Vertica JDBC driver
-RUN wget -c
https://repo1.maven.org/maven2/com/vertica/jdbc/vertica-jdbc/23.4.0-0/vertica-jdbc-23.4.0-0.jar
-O /opt/hop/lib/jdbc/vertica-jdbc-23.4.0-0.jar
+ && ./hop-conf.sh --generate-fat-jar=/tmp/hop-fatjar.jar \
+ # Download the Vertica JDBC driver
+ && wget -c
https://repo1.maven.org/maven2/com/vertica/jdbc/vertica-jdbc/23.4.0-0/vertica-jdbc-23.4.0-0.jar
-O /opt/hop/lib/jdbc/vertica-jdbc-23.4.0-0.jar
# make volume available so that hop pipeline and workflow files can be
provided easily
VOLUME ["/files"]
diff --git a/docker/integration-tests/integration-tests-azure.yaml
b/docker/integration-tests/integration-tests-azure.yaml
index 57e2bc8ca5..a6c34edbb7 100644
--- a/docker/integration-tests/integration-tests-azure.yaml
+++ b/docker/integration-tests/integration-tests-azure.yaml
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-version: "2.4"
services:
integration_test_azure:
extends:
diff --git a/docker/integration-tests/integration-tests-base-spark.yaml
b/docker/integration-tests/integration-tests-base-spark.yaml
index d6de110931..599ed550ed 100644
--- a/docker/integration-tests/integration-tests-base-spark.yaml
+++ b/docker/integration-tests/integration-tests-base-spark.yaml
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-version: '2.4'
services:
integration_test:
build:
diff --git a/docker/integration-tests/integration-tests-base.yaml
b/docker/integration-tests/integration-tests-base.yaml
index a6a4b53e41..02e4d893bf 100644
--- a/docker/integration-tests/integration-tests-base.yaml
+++ b/docker/integration-tests/integration-tests-base.yaml
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-version: '2.4'
services:
integration_test:
image: hop-base-image
diff --git a/docker/integration-tests/integration-tests-cassandra.yaml
b/docker/integration-tests/integration-tests-cassandra.yaml
index 918b739d9c..38f6c13600 100644
--- a/docker/integration-tests/integration-tests-cassandra.yaml
+++ b/docker/integration-tests/integration-tests-cassandra.yaml
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-version: '2.4'
services:
integration_test_cassandra:
extends:
diff --git a/docker/integration-tests/integration-tests-database.yaml
b/docker/integration-tests/integration-tests-database.yaml
index 2aa1f4e28e..9df8e62398 100644
--- a/docker/integration-tests/integration-tests-database.yaml
+++ b/docker/integration-tests/integration-tests-database.yaml
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-version: '2.4'
services:
integration_test_database:
extends:
diff --git a/docker/integration-tests/integration-tests-flink.yaml
b/docker/integration-tests/integration-tests-flink.yaml
index ae6eddb5ee..72db39efd7 100644
--- a/docker/integration-tests/integration-tests-flink.yaml
+++ b/docker/integration-tests/integration-tests-flink.yaml
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-version: '2.4'
services:
integration_test_flink:
extends:
diff --git a/docker/integration-tests/integration-tests-hop_server.yaml
b/docker/integration-tests/integration-tests-hop_server.yaml
index e6a3156324..21999d3653 100644
--- a/docker/integration-tests/integration-tests-hop_server.yaml
+++ b/docker/integration-tests/integration-tests-hop_server.yaml
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-version: '2.4'
services:
integration_test_hop_server:
extends:
diff --git a/docker/integration-tests/integration-tests-http.yaml
b/docker/integration-tests/integration-tests-http.yaml
index 082fdf7c11..b589cc7fef 100644
--- a/docker/integration-tests/integration-tests-http.yaml
+++ b/docker/integration-tests/integration-tests-http.yaml
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-version: '2.4'
services:
integration_test_http:
extends:
diff --git a/docker/integration-tests/integration-tests-kafka.yaml
b/docker/integration-tests/integration-tests-kafka.yaml
index 27a034dbeb..f033c1c78d 100644
--- a/docker/integration-tests/integration-tests-kafka.yaml
+++ b/docker/integration-tests/integration-tests-kafka.yaml
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-version: '2.4'
services:
integration_test_kafka:
extends:
diff --git a/docker/integration-tests/integration-tests-mail.yaml
b/docker/integration-tests/integration-tests-mail.yaml
index 43359bf5ed..87e3a08ca4 100644
--- a/docker/integration-tests/integration-tests-mail.yaml
+++ b/docker/integration-tests/integration-tests-mail.yaml
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-version: '2.4'
services:
integration_test_mail:
extends:
diff --git a/docker/integration-tests/integration-tests-monetdb.yaml
b/docker/integration-tests/integration-tests-monetdb.yaml
index 56ca6049c5..2f5188637e 100644
--- a/docker/integration-tests/integration-tests-monetdb.yaml
+++ b/docker/integration-tests/integration-tests-monetdb.yaml
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-version: '2.4'
services:
integration_test_monetdb:
extends:
diff --git a/docker/integration-tests/integration-tests-mongo.yaml
b/docker/integration-tests/integration-tests-mongo.yaml
index 1c198d0bfb..9d5951e57b 100644
--- a/docker/integration-tests/integration-tests-mongo.yaml
+++ b/docker/integration-tests/integration-tests-mongo.yaml
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-version: '3.5'
services:
integration_test_mongo:
extends:
diff --git a/docker/integration-tests/integration-tests-neo4j.yaml
b/docker/integration-tests/integration-tests-neo4j.yaml
index c00a4f9f68..21f77e39c5 100644
--- a/docker/integration-tests/integration-tests-neo4j.yaml
+++ b/docker/integration-tests/integration-tests-neo4j.yaml
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-version: '2.4'
services:
integration_test_neo4j:
extends:
diff --git a/docker/integration-tests/integration-tests-snmp.yaml
b/docker/integration-tests/integration-tests-snmp.yaml
index e54150af1c..989a8b2b03 100644
--- a/docker/integration-tests/integration-tests-snmp.yaml
+++ b/docker/integration-tests/integration-tests-snmp.yaml
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-version: '2.4'
services:
integration_test_snmp:
extends:
diff --git a/docker/integration-tests/integration-tests-spark.yaml
b/docker/integration-tests/integration-tests-spark.yaml
index 88b1779490..36701d1050 100644
--- a/docker/integration-tests/integration-tests-spark.yaml
+++ b/docker/integration-tests/integration-tests-spark.yaml
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-version: '2.4'
services:
integration_test_spark:
extends:
diff --git a/docker/integration-tests/integration-tests-vertica.yaml
b/docker/integration-tests/integration-tests-vertica.yaml
index f0a90018bc..b6301cacec 100644
--- a/docker/integration-tests/integration-tests-vertica.yaml
+++ b/docker/integration-tests/integration-tests-vertica.yaml
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-version: '3.4'
services:
integration_test_vertica:
extends:
diff --git a/docker/integration-tests/integration-tests-xml.yaml
b/docker/integration-tests/integration-tests-xml.yaml
index e467362d78..3026a0a1ba 100644
--- a/docker/integration-tests/integration-tests-xml.yaml
+++ b/docker/integration-tests/integration-tests-xml.yaml
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-version: '2.4'
services:
integration_test_xml:
extends:
diff --git a/integration-tests/scripts/run-tests-docker.sh
b/integration-tests/scripts/run-tests-docker.sh
index a0b448b9f7..9a8f843e77 100755
--- a/integration-tests/scripts/run-tests-docker.sh
+++ b/integration-tests/scripts/run-tests-docker.sh
@@ -20,6 +20,7 @@
CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
DOCKER_FILES_DIR="$(cd ${CURRENT_DIR}/../../docker/integration-tests/ && pwd)"
+EXECUTED_COMPOSE_FILES=("${DOCKER_FILES_DIR}/integration-tests-base.yaml")
for ARGUMENT in "$@"; do
@@ -71,20 +72,22 @@ if [ -z "${KEEP_IMAGES}" ]; then
KEEP_IMAGES="false"
fi
-#Cleanup surefire reports
+# Cleanup surefire reports
rm -rf "${CURRENT_DIR}"/../surefire-reports
mkdir -p "${CURRENT_DIR}"/../surefire-reports/
-#Build base image only once
- docker compose -f ${DOCKER_FILES_DIR}/integration-tests-base.yaml build
--build-arg JENKINS_USER=${JENKINS_USER} --build-arg JENKINS_UID=${JENKINS_UID}
--build-arg JENKINS_GROUP=${JENKINS_GROUP} --build-arg
JENKINS_GID=${JENKINS_GID} --build-arg GCP_KEY_FILE=${GCP_KEY_FILE}
+# Unzip Hop
+unzip -o -q "${CURRENT_DIR}/../../assemblies/client/target/*.zip" -d
${CURRENT_DIR}/../../assemblies/client/target/
+# Build base image only once
+#docker compose -f ${DOCKER_FILES_DIR}/integration-tests-base.yaml build
--build-arg JENKINS_USER=${JENKINS_USER} --build-arg JENKINS_UID=${JENKINS_UID}
--build-arg JENKINS_GROUP=${JENKINS_GROUP} --build-arg
JENKINS_GID=${JENKINS_GID} --build-arg GCP_KEY_FILE=${GCP_KEY_FILE}
-#Loop over project folders
+# Loop over project folders
for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do
+
if [[ "$d" != *"scripts/" ]] && [[ "$d" != *"surefire-reports/" ]] && [[
"$d" != *"hopweb/" ]]; then
# If there is a file called disabled.txt the project is disabled
- #
if [ ! -f "$d/disabled.txt" ]; then
PROJECT_NAME=$(basename $d)
@@ -93,44 +96,47 @@ for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do
echo "project path: $d"
echo "docker compose path: ${DOCKER_FILES_DIR}"
- #Check if specific compose exists
+ # Check if specific compose exists
if [ -f "${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml" ];
then
echo "Project compose exists."
- PROJECT_NAME=${PROJECT_NAME} docker compose -f
${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml up
--abort-on-container-exit
+ EXECUTED_COMPOSE_FILES=("${EXECUTED_COMPOSE_FILES[@]}"
"${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml")
+ #PROJECT_NAME=${PROJECT_NAME} docker compose -f
${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml up
--abort-on-container-exit
else
echo "Project compose does not exists."
- PROJECT_NAME=${PROJECT_NAME} docker compose -f
${DOCKER_FILES_DIR}/integration-tests-base.yaml up --abort-on-container-exit
+ #PROJECT_NAME=${PROJECT_NAME} docker compose -f
${DOCKER_FILES_DIR}/integration-tests-base.yaml up --abort-on-container-exit
fi
fi
fi
- #create final report
+ # Create final report
if [ "${SUREFIRE_REPORT}" == "true" ]; then
- if [ ! -f
"${CURRENT_DIR}/../surefire-reports/surefile_${PROJECT_NAME}.xml" ]; then
- echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
>"${CURRENT_DIR}"/../surefire-reports/surefile_${PROJECT_NAME}.xml
- echo "<testsuite
xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\";
xsi:noNamespaceSchemaLocation=\"https://maven.apache.org/surefire/maven-surefire-plugin/xsd/surefire-test-report-3.0.xsd\";
version=\"3.0\" name=\"${PROJECT_NAME}\" time=\"0\" tests=\"0\" errors=\"0\"
skipped=\"0\" failures=\"0\">"
>>"${CURRENT_DIR}"/../surefire-reports/surefile_${PROJECT_NAME}.xml
- echo "</testsuite>"
>>"${CURRENT_DIR}"/../surefire-reports/surefile_${PROJECT_NAME}.xml
+ if [ ! -f
"${CURRENT_DIR}/../surefire-reports/surefile_${PROJECT_NAME}.xml" ]; then
+ echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
>"${CURRENT_DIR}"/../surefire-reports/surefile_${PROJECT_NAME}.xml
+ echo "<testsuite xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\";
xsi:noNamespaceSchemaLocation=\"https://maven.apache.org/surefire/maven-surefire-plugin/xsd/surefire-test-report-3.0.xsd\";
version=\"3.0\" name=\"${PROJECT_NAME}\" time=\"0\" tests=\"1\" errors=\"1\"
skipped=\"0\" failures=\"0\">"
>>"${CURRENT_DIR}"/../surefire-reports/surefile_${PROJECT_NAME}.xml
+ echo "<testcase name=\"environment_setup\" time=\"1\"><failure
type=\"could not start\"></failure><system-out><![CDATA[ Could not start docker
environment ]]></system-out><system-err><![CDATA[ Could not start docker
environment ]]></system-err></testcase>"
>>"${CURRENT_DIR}"/../surefire-reports/surefile_${PROJECT_NAME}.xml
+ echo "</testsuite>"
>>"${CURRENT_DIR}"/../surefire-reports/surefile_${PROJECT_NAME}.xml
fi
fi
done
echo "Keep images value: ${KEEP_IMAGES}"
-#Cleanup all images
+# Cleanup all images
if [ ! "${KEEP_IMAGES}" == "true" ]; then
- for d in ${DOCKER_FILES_DIR}/integration-tests-*.yaml; do
- docker compose -f $d down --rmi all --remove-orphans
+ for d in "${EXECUTED_COMPOSE_FILES[@]}"; do
+ echo "Removing: " $d
+ PROJECT_NAME="" docker compose -f $d down --rmi all --remove-orphans
done
fi
-#Print Final Results
+# Print Final Results
if [ -f "${CURRENT_DIR}/../surefire-reports/passed_tests" ]; then
echo -e "\033[1;32mPassed tests:"
PASSED_TESTS="$(cat ../surefire-reports/passed_tests)"
echo -e "\033[1;32m${PASSED_TESTS}"
fi
if [ -f "${CURRENT_DIR}/../surefire-reports/failed_tests" ]; then
- echo -e "\033[1;91mFailed tests:"
- FAILED_TESTS="$(cat ../surefire-reports/failed_tests)"
- echo -e "\033[1;91m${FAILED_TESTS}"
+ echo -e "\033[1;91mFailed tests:"
+ FAILED_TESTS="$(cat ../surefire-reports/failed_tests)"
+ echo -e "\033[1;91m${FAILED_TESTS}"
fi
