This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch integration-tests
in repository https://gitbox.apache.org/repos/asf/incubator-hop.git
The following commit(s) were added to refs/heads/integration-tests by this push:
new 0e944aa HOP-2212 (#436)
0e944aa is described below
commit 0e944aa374a979f13a7448494827e86e07f31aec
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Sat Dec 5 15:51:05 2020 +0100
HOP-2212 (#436)
* add initial integration tests to hop
* HOP-2212
---
Dockerfile.unit-tests | 0
Jenkinsfile.daily | 112 +++++++++++++++++++
integration-tests/HOP-2212/main.hpl | 101 +++++++++++++++++
integration-tests/HOP-2212/main.hwf | 123 +++++++++++++++++++++
.../metadata/pipeline-run-configuration/local.json | 17 +++
.../metadata/workflow-run-configuration/local.json | 9 ++
integration-tests/HOP-2212/project-config.json | 9 ++
integration-tests/scripts/run-tests.sh | 100 +++++++++++++++++
8 files changed, 471 insertions(+)
diff --git a/Dockerfile.unit-tests b/Dockerfile.unit-tests
new file mode 100644
index 0000000..e69de29
diff --git a/Jenkinsfile.daily b/Jenkinsfile.daily
new file mode 100644
index 0000000..6f47d31
--- /dev/null
+++ b/Jenkinsfile.daily
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+def AGENT_LABEL = env.AGENT_LABEL ?: 'ubuntu'
+def JDK_NAME = env.JDK_NAME ?: 'jdk_1.8_latest'
+def MAVEN_NAME = env.MAVEN_NAME ?: 'maven_3_latest'
+
+def MAVEN_PARAMS = "-T 2 -U -B -e -fae -V -Dmaven.compiler.fork=true
-Dsurefire.rerunFailingTestsCount=2"
+
+pipeline {
+
+ agent {
+ label AGENT_LABEL
+ }
+
+ tools {
+ jdk JDK_NAME
+ maven MAVEN_NAME
+ }
+
+ environment {
+ MAVEN_SKIP_RC = true
+ }
+
+ options {
+ buildDiscarder(
+ logRotator(artifactNumToKeepStr: '5', numToKeepStr: '10')
+ )
+ disableConcurrentBuilds()
+ }
+
+ parameters {
+ booleanParam(name: 'CLEAN', defaultValue: true, description: 'Perform
the build in clean workspace')
+ }
+
+ stages {
+ stage('Initialization') {
+ steps {
+ echo 'Building Branch: ' + env.BRANCH_NAME
+ echo 'Using PATH = ' + env.PATH
+ }
+ }
+ stage('Cleanup') {
+ steps {
+ echo 'Cleaning up the workspace'
+ deleteDir()
+ }
+ }
+ stage('Checkout') {
+ steps {
+ echo 'Checking out branch ' + env.BRANCH_NAME
+ checkout scm
+ }
+ }
+ stage('Build') {
+ when {
+ branch 'integration-tests'
+ }
+ steps {
+ echo 'Build'
+
+ dir("local-snapshots-dir/") {
+ deleteDir()
+ }
+
+ sh "mvn $MAVEN_PARAMS clean install"
+ }
+ post {
+ always {
+ junit(testResults: '**/surefire-reports/*.xml',
allowEmptyResults: true)
+ junit(testResults: '**/failsafe-reports/*.xml',
allowEmptyResults: true)
+ }
+ }
+ }
+ stage('Code Quality') {
+ steps {
+ echo 'Checking Code Quality on SonarCloud'
+ withCredentials([string(credentialsId:
'sonarcloud-key-apache-hop', variable: 'SONAR_TOKEN')]) {
+ sh 'mvn sonar:sonar -Dsonar.host.url=https://sonarcloud.io
-Dsonar.organization=apache -Dsonar.projectKey=apache_incubator-hop
-Dsonar.branch.name=${BRANCH_NAME} -Dsonar.login=${SONAR_TOKEN}'
+ }
+ }
+ }
+
+
+ }
+ post {
+ always {
+ cleanWs()
+ emailext(
+ subject: '${DEFAULT_SUBJECT}',
+ body: '${DEFAULT_CONTENT}',
+ recipientProviders: [[$class: 'CulpritsRecipientProvider']]
+ )
+ }
+ }
+}
diff --git a/integration-tests/HOP-2212/main.hpl
b/integration-tests/HOP-2212/main.hpl
new file mode 100644
index 0000000..cb56d9e
--- /dev/null
+++ b/integration-tests/HOP-2212/main.hpl
@@ -0,0 +1,101 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<pipeline>
+ <info>
+ <name>main</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2020/11/20 10:13:03.510</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2020/11/20 10:13:03.510</modified_date>
+ <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Generate rows</from>
+ <to>Dummy (do nothing)</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Dummy (do nothing)</from>
+ <to>Abort</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Dummy (do nothing)</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>443</xloc>
+ <yloc>162</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Generate rows</name>
+ <type>RowGenerator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ </fields>
+ <limit>10</limit>
+ <never_ending>N</never_ending>
+ <interval_in_ms>5000</interval_in_ms>
+ <row_time_field>now</row_time_field>
+ <last_time_field>FiveSecondsAgo</last_time_field>
+ <attributes/>
+ <GUI>
+ <xloc>229</xloc>
+ <yloc>162</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Abort</name>
+ <type>Abort</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <row_threshold>0</row_threshold>
+ <message/>
+ <always_log_rows>Y</always_log_rows>
+ <abort_option>ABORT_WITH_ERROR</abort_option>
+ <attributes/>
+ <GUI>
+ <xloc>584</xloc>
+ <yloc>162</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/HOP-2212/main.hwf
b/integration-tests/HOP-2212/main.hwf
new file mode 100644
index 0000000..f3a59c2
--- /dev/null
+++ b/integration-tests/HOP-2212/main.hwf
@@ -0,0 +1,123 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<workflow>
+ <name>main</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2020/12/05 14:11:58.724</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2020/12/05 14:11:58.724</modified_date>
+ <parameters>
+ </parameters>
+ <pass_batchid>N</pass_batchid>
+ <actions>
+ <action>
+ <name>START</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <start>Y</start>
+ <dummy>N</dummy>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <hour>12</hour>
+ <minutes>0</minutes>
+ <weekDay>1</weekDay>
+ <DayOfMonth>1</DayOfMonth>
+ <parallel>N</parallel>
+ <nr>0</nr>
+ <xloc>327</xloc>
+ <yloc>139</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Pipeline</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/main.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <set_append_logfile>N</set_append_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <follow_abort_remote>N</follow_abort_remote>
+ <create_parent_folder>N</create_parent_folder>
+ <run_configuration>local</run_configuration>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <parallel>N</parallel>
+ <nr>0</nr>
+ <xloc>517</xloc>
+ <yloc>140</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Success</name>
+ <description/>
+ <type>SUCCESS</type>
+ <attributes/>
+ <parallel>N</parallel>
+ <nr>0</nr>
+ <xloc>744</xloc>
+ <yloc>138</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Abort workflow</name>
+ <description/>
+ <type>ABORT</type>
+ <attributes/>
+ <message/>
+ <parallel>N</parallel>
+ <nr>0</nr>
+ <xloc>512</xloc>
+ <yloc>256</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>START</from>
+ <to>Pipeline</to>
+ <from_nr>0</from_nr>
+ <to_nr>0</to_nr>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>Pipeline</from>
+ <to>Success</to>
+ <from_nr>0</from_nr>
+ <to_nr>0</to_nr>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>Pipeline</from>
+ <to>Abort workflow</to>
+ <from_nr>0</from_nr>
+ <to_nr>0</to_nr>
+ <enabled>Y</enabled>
+ <evaluation>N</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/HOP-2212/metadata/pipeline-run-configuration/local.json
b/integration-tests/HOP-2212/metadata/pipeline-run-configuration/local.json
new file mode 100644
index 0000000..f1b301f
--- /dev/null
+++ b/integration-tests/HOP-2212/metadata/pipeline-run-configuration/local.json
@@ -0,0 +1,17 @@
+{
+ "engineRunConfiguration": {
+ "Local": {
+ "feedback_size": "50000",
+ "sample_size": "100",
+ "sample_type_in_gui": "Last",
+ "rowset_size": "10000",
+ "safe_mode": false,
+ "show_feedback": false,
+ "topo_sort": false,
+ "gather_metrics": false
+ }
+ },
+ "name": "local",
+ "configurationVariables": [],
+ "description": "Runs your pipelines locally with the standard local Hop
pipeline engine"
+}
\ No newline at end of file
diff --git
a/integration-tests/HOP-2212/metadata/workflow-run-configuration/local.json
b/integration-tests/HOP-2212/metadata/workflow-run-configuration/local.json
new file mode 100644
index 0000000..e37a930
--- /dev/null
+++ b/integration-tests/HOP-2212/metadata/workflow-run-configuration/local.json
@@ -0,0 +1,9 @@
+{
+ "engineRunConfiguration": {
+ "Local": {
+ "safe_mode": false
+ }
+ },
+ "name": "local",
+ "description": "Runs your workflows locally with the standard local Hop
workflow engine"
+}
\ No newline at end of file
diff --git a/integration-tests/HOP-2212/project-config.json
b/integration-tests/HOP-2212/project-config.json
new file mode 100644
index 0000000..4b92204
--- /dev/null
+++ b/integration-tests/HOP-2212/project-config.json
@@ -0,0 +1,9 @@
+{
+ "metadataBaseFolder" : "${PROJECT_HOME}/metadata/",
+ "unitTestsBasePath" : "${PROJECT_HOME}",
+ "dataSetsCsvFolder" : "${PROJECT_HOME}/datasets",
+ "enforcingExecutionInHome" : true,
+ "config" : {
+ "variables" : [ ]
+ }
+}
\ No newline at end of file
diff --git a/integration-tests/scripts/run-tests.sh
b/integration-tests/scripts/run-tests.sh
new file mode 100755
index 0000000..28df867
--- /dev/null
+++ b/integration-tests/scripts/run-tests.sh
@@ -0,0 +1,100 @@
+#!/bin/bash
+
+if [ -z "${HOP_LOCATION}" ]; then
+ HOP_LOCATION=/opt/hop
+fi
+
+#set start variables
+start_time=$SECONDS
+test_counter=0
+errors_counter=0
+skipped_counter=0
+failures_counter=0
+spacer="==========================================="
+
+#cleanup surefire report
+/dev/null > /tmp/testcases
+
+for d in ../*/ ; do
+ if [[ "$d" != *"scripts/" ]]; then
+
+ test_name=$(basename $d)
+
+ echo $spacer
+ echo "Starting Test: $test_name"
+ echo $spacer
+
+ #Increment timer and set test start time
+ test_counter=$((test_counter+1))
+
+ #Delete project first
+ $HOP_LOCATION/hop-conf.sh -pd -p $test_name
+
+ #Create New Project
+ $HOP_LOCATION/hop-conf.sh -pc -p $test_name -ph "$(readlink -f $d)"
+
+ #Find main hwf/ TODO: add hpl support when result is returned correctly
+ HOP_FILE="$(readlink -f $d/main.hwf)"
+
+ #Start time test
+ start_time_test=$SECONDS
+
+ #Run Test
+ $HOP_LOCATION/hop-run.sh -j $test_name -r "local" -f $HOP_FILE > >(tee
/tmp/test_output) 2> >(tee /tmp/test_output_err >&1)
+
+ #Capture exit code
+ exit_code=${PIPESTATUS[0]}
+
+ #Test time duration
+ test_duration=$(( SECONDS - start_time_test ))
+
+ if (( $exit_code >= 1 )) ;
+ then
+ errors_counter=$((errors_counter+1))
+ failures_counter=$((failures_counter+1))
+ #Create surefire xml failure
+ echo "<testcase name=\"$test_name\" time=\"$test_duration\">" >>
/tmp/testcases
+ echo "<failure type=\"$test_name\"></failure>" >> /tmp/testcases
+ echo "<system-out>" >> /tmp/testcases
+ cat /tmp/test_output >> /tmp/testcases
+ echo "</system-out>" >> /tmp/testcases
+ echo "<system-err>" >> /tmp/testcases
+ cat /tmp/test_output_err >> /tmp/testcases
+ echo "</system-err>" >> /tmp/testcases
+ echo "</testcase>" >> /tmp/testcases
+
+ else
+ #Create surefire xml success
+ echo "<testcase name=\"$test_name\" time=\"$test_duration\">" >>
/tmp/testcases
+ echo "<system-out>" >> /tmp/testcases
+ cat /tmp/test_output >> /tmp/testcases
+ echo "</system-out>" >> /tmp/testcases
+ echo "</testcase>" >> /tmp/testcases
+ fi
+
+ #Print results to console
+ echo $spacer
+ echo "Test Result"
+ echo $spacer
+ echo "Test duration: $test_duration"
+ echo "Test Exit Code: $exit_code"
+
+ fi
+done
+
+total_duration=$(( SECONDS - start_time ))
+
+#Print End results
+echo $spacer
+echo "Final Report"
+echo $spacer
+echo "Number of Tests: $test_counter"
+echo "Total errors: $errors_counter"
+echo "Total faliures: $failures_counter"
+echo "Total duration: $total_duration"
+
+#create final report
+echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" >> /tmp/surefire_report.xml
+echo "<testsuite xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\";
xsi:noNamespaceSchemaLocation=\"https://maven.apache.org/surefire/maven-surefire-plugin/xsd/surefire-test-report-3.0.xsd\";
version=\"3.0\" name=\"Hop Integration Tests\" time=\"$total_duration\"
tests=\"$test_counter\" errors=\"$errors_counter\" skipped=\"$skipped_counter\"
failures=\"$failures_counter\">" >> /tmp/surefire_report.xml
+cat /tmp/testcases >> /tmp/surefire_report.xml
+echo "</testsuite>" >> /tmp/surefire_report.xml
