This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch 4.x
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x by this push:
new fe03af0 PHOENIX-6146 Run precommit checks on github PRs
fe03af0 is described below
commit fe03af0c1c8c19b0908919e0ee040927d8e35980
Author: Istvan Toth <[email protected]>
AuthorDate: Thu Sep 24 10:58:01 2020 +0200
PHOENIX-6146 Run precommit checks on github PRs
---
.asf.yaml | 28 ++++++
Jenkinsfile.github | 173 ++++++++++++++++++++++++++++++++++
dev/gather_machine_environment.sh | 58 ++++++++++++
dev/jenkins_precommit_github_yetus.sh | 151 +++++++++++++++++++++++++++++
4 files changed, 410 insertions(+)
diff --git a/.asf.yaml b/.asf.yaml
new file mode 100644
index 0000000..22e3eab
--- /dev/null
+++ b/.asf.yaml
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file controls the integration of the Phoenix project with ASF
+# infrastructure. Refer to
+# https://cwiki.apache.org/confluence/display/INFRA/git+-+.asf.yaml+features
+# for details. Be careful when changing the contents of this file since it
+# may affect many developers of the project and make sure to discuss the
+# changes with dev@ before committing.
+
+notifications:
+ commits: [email protected]
+ issues: [email protected]
+ pullrequests: [email protected]
+ jira_options: link label comment
diff --git a/Jenkinsfile.github b/Jenkinsfile.github
new file mode 100644
index 0000000..9221e02
--- /dev/null
+++ b/Jenkinsfile.github
@@ -0,0 +1,173 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+pipeline {
+
+ //FIXME convert this to Matrix job
+ agent {
+ label 'Hadoop'
+ }
+
+ options {
+ // N.B. this is per-branch, which means per PR
+ disableConcurrentBuilds()
+ buildDiscarder(logRotator(numToKeepStr: '15'))
+ timeout (time: 9, unit: 'HOURS')
+ timestamps()
+ skipDefaultCheckout()
+ }
+
+ environment {
+ SRC_REL = 'src'
+ PATCH_REL = 'output'
+ YETUS_REL = 'yetus'
+ DOCKERFILE_REL = "${SRC_REL}/dev/docker/Dockerfile.yetus"
+ YETUS_DRIVER_REL = "${SRC_REL}/dev/jenkins_precommit_github_yetus.sh"
+ // Branch or tag name. Yetus release tags are 'rel/X.Y.Z'
+ YETUS_VERSION = 'rel/0.12.0'
+ PLUGINS= 'all,-findbugs,-gitlab'
+ //GENERAL_CHECK_PLUGINS =
'all,-compile,-javac,-javadoc,-jira,-shadedjars,-unit'
+ //JDK_SPECIFIC_PLUGINS =
'compile,github,htmlout,javac,javadoc,maven,mvninstall,shadedjars,unit'
+ // output from surefire; sadly the archive function in yetus only
works on file names.
+ ARCHIVE_PATTERN_LIST =
'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
+ // These tests currently have known failures. Once they burn down to
0, remove from here so that new problems will cause a failure.
+ //TESTS_FILTER =
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
+ TESTS_FILTER = 'dummy'
+ //EXCLUDE_TESTS_URL =
"${JENKINS_URL}/job/HBase/job/HBase-Find-Flaky-Tests/job/${CHANGE_TARGET}/lastSuccessfulBuild/artifact/excludes"
+
+ // a global view of paths. parallel stages can land on the same host
concurrently, so each
+ // stage works in its own subdirectory. there is an "output" under
each of these
+ // directories, which we retrieve after the build is complete.
+ WORKDIR_REL_GENERAL_CHECK = 'yetus-general-check'
+ //WORKDIR_REL_JDK8_HADOOP3_CHECK = 'yetus-jdk8-hadoop3-check'
+ //WORKDIR_REL_JDK11_HADOOP3_CHECK = 'yetus-jdk11-hadoop3-check'
+ GITHUB_USE_TOKEN = 'true'
+ }
+
+ parameters {
+ booleanParam(name: 'DEBUG',
+ defaultValue: false,
+ description: 'Print extra outputs for debugging the jenkins job
and yetus')
+ }
+
+ stages {
+ stage ('precommit checks') {
+//Disabled while don't have actual parallel tasks, as this ties up an extra
executor
+// parallel {
+// stage ('yetus general check') {
+// agent {
+// node {
+// label 'Hadoop'
+// }
+// }
+ environment {
+ // customized per parallel stage
+ //PLUGINS = "${GENERAL_CHECK_PLUGINS}"
+ SET_JAVA_HOME = '/usr/lib/jvm/java-8'
+ WORKDIR_REL = "${WORKDIR_REL_GENERAL_CHECK}"
+ // identical for all parallel stages
+ WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
+ YETUSDIR = "${WORKDIR}/${YETUS_REL}"
+ SOURCEDIR = "${WORKDIR}/${SRC_REL}"
+ PATCHDIR = "${WORKDIR}/${PATCH_REL}"
+ BUILD_URL_ARTIFACTS =
"artifact/${WORKDIR_REL}/${PATCH_REL}"
+ DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
+ YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
+ }
+ steps {
+ dir("${SOURCEDIR}") {
+ checkout scm
+ }
+ dir("${YETUSDIR}") {
+ checkout([
+ $class : 'GitSCM',
+ branches : [[name: "${YETUS_VERSION}"]],
+ userRemoteConfigs: [[url:
'https://github.com/apache/yetus.git']]]
+ )
+ }
+ dir("${WORKDIR}") {
+ withCredentials([
+ usernamePassword(
+ credentialsId:
'c06659ac-4c77-499f-9b9d-485f1b58792c',
+ passwordVariable: 'GITHUB_PASSWORD',
+ usernameVariable: 'GITHUB_USER'
+ )]) {
+ sh label: 'test-patch', script: '''#!/bin/bash
-e
+ hostname -a ; pwd ; ls -la
+ printenv 2>&1 | sort
+ echo "[INFO] Launching Yetus via
${YETUS_DRIVER}"
+ "${YETUS_DRIVER}"
+ '''
+ }
+ }
+ }
+ post {
+ always {
+ // Has to be relative to WORKSPACE.
+ archiveArtifacts artifacts:
"${WORKDIR_REL}/${PATCH_REL}/*", excludes:
"${WORKDIR_REL}/${PATCH_REL}/precommit"
+ archiveArtifacts artifacts:
"${WORKDIR_REL}/${PATCH_REL}/**/*", excludes:
"${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
+ publishHTML target: [
+ allowMissing: true,
+ keepAll: true,
+ alwaysLinkToLastBuild: true,
+ // Has to be relative to WORKSPACE
+ reportDir: "${WORKDIR_REL}/${PATCH_REL}",
+ reportFiles: 'report.html',
+ reportName: 'PR General Check Report'
+ ]
+ }
+ // Jenkins pipeline jobs fill slaves on PRs without
this :(
+ cleanup() {
+ script {
+ sh label: 'Cleanup workspace', script:
'''#!/bin/bash -e
+ # See YETUS-764
+ if [ -f "${PATCHDIR}/pidfile.txt" ]; then
+ echo "test-patch process appears to
still be running: killing"
+ kill `cat "${PATCHDIR}/pidfile.txt"` ||
true
+ sleep 10
+ fi
+ if [ -f "${PATCHDIR}/cidfile.txt" ]; then
+ echo "test-patch container appears to
still be running: killing"
+ docker kill `cat
"${PATCHDIR}/cidfile.txt"` || true
+ fi
+ # See HADOOP-13951
+ chmod -R u+rxw "${WORKSPACE}"
+ '''
+ dir ("${WORKDIR}") {
+ deleteDir()
+ }
+ }
+ }
+ }
+ //}
+ //}
+ }
+ }
+
+ post {
+ // Jenkins pipeline jobs fill slaves on PRs without this :(
+ cleanup() {
+ script {
+ sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
+ # See HADOOP-13951
+ chmod -R u+rxw "${WORKSPACE}"
+ '''
+ deleteDir()
+ }
+ }
+ }
+}
diff --git a/dev/gather_machine_environment.sh
b/dev/gather_machine_environment.sh
new file mode 100755
index 0000000..265a328
--- /dev/null
+++ b/dev/gather_machine_environment.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# SHELLDOC-IGNORE
+
+set -e
+function usage {
+ echo "Usage: ${0} /path/for/output/dir"
+ echo ""
+ echo " Gather info about a build machine that test harnesses should poll
before running."
+ echo " presumes you'll then archive the passed output dir."
+
+ exit 1
+}
+
+if [ "$#" -lt 1 ]; then
+ usage
+fi
+
+
+declare output=$1
+
+if [ ! -d "${output}" ] || [ ! -w "${output}" ]; then
+ echo "Specified output directory must exist and be writable." >&2
+ exit 1
+fi
+
+echo "getting machine specs, find in ${BUILD_URL}/artifact/${output}/"
+echo "JAVA_HOME: ${JAVA_HOME}" >"${output}/java_home" 2>&1 || true
+ls -l "${JAVA_HOME}" >"${output}/java_home_ls" 2>&1 || true
+echo "MAVEN_HOME: ${MAVEN_HOME}" >"${output}/mvn_home" 2>&1 || true
+mvn --offline --version >"${output}/mvn_version" 2>&1 || true
+cat /proc/cpuinfo >"${output}/cpuinfo" 2>&1 || true
+cat /proc/meminfo >"${output}/meminfo" 2>&1 || true
+cat /proc/diskstats >"${output}/diskstats" 2>&1 || true
+cat /sys/block/sda/stat >"${output}/sys-block-sda-stat" 2>&1 || true
+df -h >"${output}/df-h" 2>&1 || true
+ps -Aww >"${output}/ps-Aww" 2>&1 || true
+ifconfig -a >"${output}/ifconfig-a" 2>&1 || true
+lsblk -ta >"${output}/lsblk-ta" 2>&1 || true
+lsblk -fa >"${output}/lsblk-fa" 2>&1 || true
+ulimit -a >"${output}/ulimit-a" 2>&1 || true
+uptime >"${output}/uptime" 2>&1 || true
diff --git a/dev/jenkins_precommit_github_yetus.sh
b/dev/jenkins_precommit_github_yetus.sh
new file mode 100755
index 0000000..f607748
--- /dev/null
+++ b/dev/jenkins_precommit_github_yetus.sh
@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+# place ourselves in the directory containing the phoenix and yetus checkouts
+cd "$(dirname "$0")/../.."
+echo "executing from $(pwd)"
+
+if [[ "true" = "${DEBUG}" ]]; then
+ set -x
+ printenv 2>&1 | sort
+fi
+
+declare -i missing_env=0
+declare -a required_envs=(
+ # these ENV variables define the required API with Jenkinsfile_GitHub
+ "ARCHIVE_PATTERN_LIST"
+ "BUILD_URL_ARTIFACTS"
+ "DOCKERFILE"
+ "GITHUB_PASSWORD"
+ "GITHUB_USER"
+ "PATCHDIR"
+ "PLUGINS"
+ "SET_JAVA_HOME"
+ "SOURCEDIR"
+ "TESTS_FILTER"
+ "YETUSDIR"
+)
+# Validate params
+for required_env in "${required_envs[@]}"; do
+ if [ -z "${!required_env}" ]; then
+ echo "[ERROR] Required environment variable '${required_env}' is not set."
+ missing_env=${missing_env}+1
+ fi
+done
+
+if [ ${missing_env} -gt 0 ]; then
+ echo "[ERROR] Please set the required environment variables before invoking.
If this error is " \
+ "on Jenkins, then please file a JIRA about the error."
+ exit 1
+fi
+
+# TODO (HBASE-23900): cannot assume test-patch runs directly from sources
+TESTPATCHBIN="${YETUSDIR}/precommit/src/main/shell/test-patch.sh"
+
+# this must be clean for every run
+rm -rf "${PATCHDIR}"
+mkdir -p "${PATCHDIR}"
+
+# Checking on H* machine nonsense
+mkdir "${PATCHDIR}/machine"
+"${SOURCEDIR}/dev/gather_machine_environment.sh" "${PATCHDIR}/machine"
+
+# If CHANGE_URL is set (e.g., Github Branch Source plugin), process it.
+# Otherwise exit, because we don't want Phoenix to do a
+# full build. We wouldn't normally do this check for smaller
+# projects. :)
+if [[ -z "${CHANGE_URL}" ]]; then
+ echo "Full build skipped" > "${PATCHDIR}/report.html"
+ exit 0
+fi
+# enable debug output for yetus
+if [[ "true" = "${DEBUG}" ]]; then
+ YETUS_ARGS+=("--debug")
+fi
+# If we're doing docker, make sure we don't accidentally pollute the image
with a host java path
+if [ -n "${JAVA_HOME}" ]; then
+ unset JAVA_HOME
+fi
+YETUS_ARGS+=("--ignore-unknown-options=true")
+YETUS_ARGS+=("--patch-dir=${PATCHDIR}")
+# where the source is located
+YETUS_ARGS+=("--basedir=${SOURCEDIR}")
+# our project defaults come from a personality file
+# which will get loaded automatically by setting the project name
+YETUS_ARGS+=("--project=phoenix")
+# lots of different output formats
+YETUS_ARGS+=("--brief-report-file=${PATCHDIR}/brief.txt")
+YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt")
+YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html")
+# enable writing back to Github
+if [[ "true" = "${GITHUB_USE_TOKEN}" ]]; then
+ YETUS_ARGS+=("--github-token=${GITHUB_PASSWORD}")
+else
+ YETUS_ARGS+=("--github-user=${GITHUB_USER}")
+ YETUS_ARGS+=("--github-password=${GITHUB_PASSWORD}")
+fi
+# auto-kill any surefire stragglers during unit test runs
+YETUS_ARGS+=("--reapermode=kill")
+# set relatively high limits for ASF machines
+# changing these to higher values may cause problems
+# with other jobs on systemd-enabled machines
+YETUS_ARGS+=("--dockermemlimit=20g")
+# -1 spotbugs issues that show up prior to the patch being applied
+#YETUS_ARGS+=("--spotbugs-strict-precheck")
+# rsync these files back into the archive dir
+YETUS_ARGS+=("--archive-list=${ARCHIVE_PATTERN_LIST}")
+# URL for user-side presentation in reports and such to our artifacts
+YETUS_ARGS+=("--build-url-artifacts=${BUILD_URL_ARTIFACTS}")
+# plugins to enable
+YETUS_ARGS+=("--plugins=${PLUGINS},-findbugs")
+# run in docker mode and specifically point to our
+# Dockerfile since we don't want to use the auto-pulled version.
+YETUS_ARGS+=("--docker")
+YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
+YETUS_ARGS+=("--mvn-custom-repos")
+YETUS_ARGS+=("--java-home=${SET_JAVA_HOME}")
+YETUS_ARGS+=("--whitespace-eol-ignore-list=.*/generated/.*")
+YETUS_ARGS+=("--whitespace-tabs-ignore-list=.*/generated/.*")
+YETUS_ARGS+=("--tests-filter=${TESTS_FILTER}")
+YETUS_ARGS+=("--personality=${SOURCEDIR}/dev/phoenix-personality.sh")
+#YETUS_ARGS+=("--quick-hadoopcheck")
+#YETUS_ARGS+=("--skip-errorprone")
+# effectively treat dev-support as a custom maven module
+YETUS_ARGS+=("--skip-dirs=dev")
+# For testing with specific hadoop version. Activates corresponding profile in
maven runs.
+#if [[ -n "${HADOOP_PROFILE}" ]]; then
+# # Master has only Hadoop3 support. We don't need to activate any profile.
+# # The Jenkinsfile should not attempt to run any Hadoop2 tests.
+# if [[ "${BRANCH_NAME}" =~ branch-2* ]]; then
+# YETUS_ARGS+=("--hadoop-profile=${HADOOP_PROFILE}")
+# fi
+#fi
+if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then
+ YETUS_ARGS+=("--exclude-tests-url=${EXCLUDE_TESTS_URL}")
+fi
+# help keep the ASF boxes clean
+YETUS_ARGS+=("--sentinel")
+# use emoji vote so it is easier to find the broken line
+YETUS_ARGS+=("--github-use-emoji-vote")
+
+echo "Launching yetus with command line:"
+echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}"
+
+/usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
