http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a2e360/dev-support/test-patch.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh index cd91a5c..e1dadd2 100755 --- a/dev-support/test-patch.sh +++ b/dev-support/test-patch.sh @@ -14,14 +14,41 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Make sure that bash version meets the pre-requisite + +if [[ -z "${BASH_VERSINFO}" ]] \ + || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \ + || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then + echo "bash v3.2+ is required. Sorry." + exit 1 +fi + ### BUILD_URL is set by Hudson if it is run by patch process this="${BASH_SOURCE-$0}" BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) -CWD=$(pwd) +STARTINGDIR=$(pwd) USER_PARAMS=("$@") GLOBALTIMER=$(date +"%s") +# global arrays +declare -a MAVEN_ARGS=("--batch-mode") +declare -a ANT_ARGS=("-noinput") +declare -a TP_HEADER +declare -a TP_VOTE_TABLE +declare -a TP_TEST_TABLE +declare -a TP_FOOTER_TABLE +declare -a MODULE_STATUS +declare -a MODULE_STATUS_TIMER +declare -a MODULE_STATUS_MSG +declare -a MODULE_STATUS_LOG +declare -a MODULE + +TP_HEADER_COUNTER=0 +TP_VOTE_COUNTER=0 +TP_TEST_COUNTER=0 +TP_FOOTER_COUNTER=0 + ## @description Setup the default global variables ## @audience public ## @stability stable @@ -33,10 +60,15 @@ function setup_defaults else MVN=${MAVEN_HOME}/bin/mvn fi - # This parameter needs to be kept as an array - MAVEN_ARGS=() - PROJECT_NAME=hadoop + if [[ -z "${ANT_HOME:-}" ]]; then + ANT=ant + else + ANT=${ANT_HOME}/bin/ant + fi + + PROJECT_NAME=yetus + DOCKERFILE="${BINDIR}/test-patch-docker/Dockerfile-startstub" HOW_TO_CONTRIBUTE="https://wiki.apache.org/hadoop/HowToContribute"; JENKINS=false BASEDIR=$(pwd) @@ -44,13 +76,15 @@ function setup_defaults USER_PLUGIN_DIR="" LOAD_SYSTEM_PLUGINS=true + ALLOWSUMMARIES=true - FINDBUGS_HOME=${FINDBUGS_HOME:-} - FINDBUGS_WARNINGS_FAIL_PRECHECK=false + DOCKERSUPPORT=false ECLIPSE_HOME=${ECLIPSE_HOME:-} BUILD_NATIVE=${BUILD_NATIVE:-true} PATCH_BRANCH="" - PATCH_BRANCH_DEFAULT="trunk" + PATCH_BRANCH_DEFAULT="master" + + #shellcheck disable=SC2034 CHANGED_MODULES="" USER_MODULE_LIST="" OFFLINE=false @@ -58,50 +92,40 @@ function setup_defaults REEXECED=false RESETREPO=false ISSUE="" - ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$' + ISSUE_RE='^(YETUS)-[0-9]+$' TIMER=$(date +"%s") PATCHURL="" - OSTYPE=$(uname -s) + BUILDTOOL=maven + BUGSYSTEM=jira + JDK_TEST_LIST="javac javadoc unit" + GITDIFFLINES="${PATCH_DIR}/gitdifflines.txt" + GITDIFFCONTENT="${PATCH_DIR}/gitdiffcontent.txt" # Solaris needs POSIX, not SVID case ${OSTYPE} in SunOS) - PS=${PS:-ps} AWK=${AWK:-/usr/xpg4/bin/awk} SED=${SED:-/usr/xpg4/bin/sed} WGET=${WGET:-wget} GIT=${GIT:-git} - EGREP=${EGREP:-/usr/xpg4/bin/egrep} GREP=${GREP:-/usr/xpg4/bin/grep} - PATCH=${PATCH:-patch} + PATCH=${PATCH:-/usr/gnu/bin/patch} DIFF=${DIFF:-/usr/gnu/bin/diff} - JIRACLI=${JIRA:-jira} FILE=${FILE:-file} ;; *) - PS=${PS:-ps} AWK=${AWK:-awk} SED=${SED:-sed} WGET=${WGET:-wget} GIT=${GIT:-git} - EGREP=${EGREP:-egrep} GREP=${GREP:-grep} PATCH=${PATCH:-patch} DIFF=${DIFF:-diff} - JIRACLI=${JIRA:-jira} FILE=${FILE:-file} ;; esac - declare -a JIRA_COMMENT_TABLE - declare -a JIRA_FOOTER_TABLE - declare -a JIRA_HEADER - declare -a JIRA_TEST_TABLE - - JFC=0 - JTC=0 - JTT=0 RESULT=0 } @@ -110,7 +134,7 @@ function setup_defaults ## @stability stable ## @replaceable no ## @param string -function hadoop_error +function yetus_error { echo "$*" 1>&2 } @@ -120,20 +144,51 @@ function hadoop_error ## @stability stable ## @replaceable no ## @param string -function hadoop_debug +function yetus_debug { - if [[ -n "${HADOOP_SHELL_SCRIPT_DEBUG}" ]]; then + if [[ -n "${TP_SHELL_SCRIPT_DEBUG}" ]]; then echo "[$(date) DEBUG]: $*" 1>&2 fi } +## @description Convert the given module name to a file fragment +## @audience public +## @stability stable +## @replaceable no +## @param module +function module_file_fragment +{ + local mod=$1 + if [[ ${mod} == . ]]; then + echo root + else + echo "$1" | tr '/' '_' | tr '\\' '_' + fi +} + +## @description Convert time in seconds to m + s +## @audience public +## @stability stable +## @replaceable no +## @param seconds +function clock_display +{ + local -r elapsed=$1 + + if [[ ${elapsed} -lt 0 ]]; then + echo "N/A" + else + printf "%3sm %02ss" $((elapsed/60)) $((elapsed%60)) + fi +} + ## @description Activate the local timer ## @audience public ## @stability stable ## @replaceable no function start_clock { - hadoop_debug "Start clock" + yetus_debug "Start clock" TIMER=$(date +"%s") } @@ -145,7 +200,7 @@ function stop_clock { local -r stoptime=$(date +"%s") local -r elapsed=$((stoptime-TIMER)) - hadoop_debug "Stop clock" + yetus_debug "Stop clock" echo ${elapsed} } @@ -158,7 +213,7 @@ function stop_global_clock { local -r stoptime=$(date +"%s") local -r elapsed=$((stoptime-GLOBALTIMER)) - hadoop_debug "Stop global clock" + yetus_debug "Stop global clock" echo ${elapsed} } @@ -178,10 +233,10 @@ function offset_clock ## @stability stable ## @replaceable no ## @param string -function add_jira_header +function add_header_line { - JIRA_HEADER[${JHC}]="| $* |" - JHC=$(( JHC+1 )) + TP_HEADER[${TP_HEADER_COUNTER}]="$*" + ((TP_HEADER_COUNTER=TP_HEADER_COUNTER+1 )) } ## @description Add to the output table. If the first parameter is a number @@ -199,88 +254,152 @@ function add_jira_header ## @param subsystem ## @param string ## @return Elapsed time display -function add_jira_table +function add_vote_table { local value=$1 local subsystem=$2 shift 2 - local color - local calctime=0 - + local calctime local -r elapsed=$(stop_clock) - if [[ ${elapsed} -lt 0 ]]; then - calctime="N/A" + yetus_debug "add_vote_table ${value} ${subsystem} ${*}" + + calctime=$(clock_display "${elapsed}") + + if [[ ${value} == "1" ]]; then + value="+1" + fi + + if [[ -z ${value} ]]; then + TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="| | ${subsystem} | | ${*:-} |" else - printf -v calctime "%3sm %02ss" $((elapsed/60)) $((elapsed%60)) + TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="| ${value} | ${subsystem} | ${calctime} | $* |" fi + ((TP_VOTE_COUNTER=TP_VOTE_COUNTER+1)) +} - echo "" - echo "Elapsed time: ${calctime}" - echo "" +## @description Report the JVM version of the given directory +## @stability stable +## @audience private +## @replaceable yes +## @params directory +## @returns version +function report_jvm_version +{ + #shellcheck disable=SC2016 + "${1}/bin/java" -version 2>&1 | head -1 | ${AWK} '{print $NF}' | tr -d \" +} - case ${value} in - 1|+1) - value="+1" - color="green" - ;; - -1) - color="red" - ;; - 0) - color="blue" - ;; - null) - ;; - esac +## @description Verify if a given test is multijdk +## @audience public +## @stability stable +## @replaceable yes +## @param test +## @return 1 = yes +## @return 0 = no +function verify_multijdk_test +{ + local i=$1 - if [[ -z ${color} ]]; then - JIRA_COMMENT_TABLE[${JTC}]="| | ${subsystem} | | ${*:-} |" - JTC=$(( JTC+1 )) - else - JIRA_COMMENT_TABLE[${JTC}]="| {color:${color}}${value}{color} | ${subsystem} | ${calctime} | $* |" - JTC=$(( JTC+1 )) + if [[ "${JDK_DIR_LIST}" == "${JAVA_HOME}" ]]; then + yetus_debug "MultiJDK not configured." + return 0 fi + + if [[ ${JDK_TEST_LIST} =~ $i ]]; then + yetus_debug "${i} is in ${JDK_TEST_LIST} and MultiJDK configured." + return 1 + fi + return 0 } -## @description Put the final environment information at the bottom +## @description Absolute path the JDK_DIR_LIST and JAVA_HOME. +## @description if JAVA_HOME is in JDK_DIR_LIST, it is positioned last +## @stability stable +## @audience private +## @replaceable yes +function fullyqualifyjdks +{ + local i + local jdkdir + local tmplist + + JAVA_HOME=$(cd -P -- "${JAVA_HOME}" >/dev/null && pwd -P) + + for i in ${JDK_DIR_LIST}; do + jdkdir=$(cd -P -- "${i}" >/dev/null && pwd -P) + if [[ ${jdkdir} != "${JAVA_HOME}" ]]; then + tmplist="${tmplist} ${jdkdir}" + fi + done + + JDK_DIR_LIST="${tmplist} ${JAVA_HOME}" + JDK_DIR_LIST=${JDK_DIR_LIST/ } +} + +## @description Put the opening environment information at the bottom ## @description of the footer table ## @stability stable ## @audience private ## @replaceable yes -function close_jira_footer +function prepopulate_footer { # shellcheck disable=SC2016 - local -r javaversion=$("${JAVA_HOME}/bin/java" -version 2>&1 | head -1 | ${AWK} '{print $NF}' | tr -d \") + local javaversion + local listofjdks local -r unamea=$(uname -a) + local i + + add_footer_table "uname" "${unamea}" + add_footer_table "Build tool" "${BUILDTOOL}" + + if [[ -n ${PERSONALITY} ]]; then + add_footer_table "Personality" "${PERSONALITY}" + fi - add_jira_footer "Java" "${javaversion}" - add_jira_footer "uname" "${unamea}" + javaversion=$(report_jvm_version "${JAVA_HOME}") + add_footer_table "Default Java" "${javaversion}" + if [[ -n ${JDK_DIR_LIST} + && ${JDK_DIR_LIST} != "${JAVA_HOME}" ]]; then + for i in ${JDK_DIR_LIST}; do + javaversion=$(report_jvm_version "${i}") + listofjdks="${listofjdks} ${i}:${javaversion}" + done + add_footer_table "Multi-JDK versions" "${listofjdks}" + fi +} + +## @description Put docker stats in various tables +## @stability stable +## @audience private +## @replaceable yes +function finish_docker_stats +{ + if [[ ${DOCKERMODE} == true ]]; then + # DOCKER_VERSION is set by our creator. + add_footer_table "Docker" "${DOCKER_VERSION}" + fi } ## @description Put the final elapsed time at the bottom of the table. ## @audience private ## @stability stable ## @replaceable no -function close_jira_table +function finish_vote_table { local -r elapsed=$(stop_global_clock) + local calctime - if [[ ${elapsed} -lt 0 ]]; then - calctime="N/A" - else - printf -v calctime "%3sm %02ss" $((elapsed/60)) $((elapsed%60)) - fi + calctime=$(clock_display "${elapsed}") echo "" echo "Total Elapsed time: ${calctime}" echo "" - - JIRA_COMMENT_TABLE[${JTC}]="| | | ${calctime} | |" - JTC=$(( JTC+1 )) + TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="| | | ${calctime} | |" + ((TP_VOTE_COUNTER=TP_VOTE_COUNTER+1 )) } ## @description Add to the footer of the display. @@BASE@@ will get replaced with the @@ -291,13 +410,13 @@ function close_jira_table ## @replaceable no ## @param subsystem ## @param string -function add_jira_footer +function add_footer_table { local subsystem=$1 shift 1 - JIRA_FOOTER_TABLE[${JFC}]="| ${subsystem} | $* |" - JFC=$(( JFC+1 )) + TP_FOOTER_TABLE[${TP_FOOTER_COUNTER}]="| ${subsystem} | $* |" + ((TP_FOOTER_COUNTER=TP_FOOTER_COUNTER+1 )) } ## @description Special table just for unit test failures @@ -306,13 +425,13 @@ function add_jira_footer ## @replaceable no ## @param failurereason ## @param testlist -function add_jira_test_table +function add_test_table { local failure=$1 shift 1 - JIRA_TEST_TABLE[${JTT}]="| ${failure} | $* |" - JTT=$(( JTT+1 )) + TP_TEST_TABLE[${TP_TEST_COUNTER}]="| ${failure} | $* |" + ((TP_TEST_COUNTER=TP_TEST_COUNTER+1 )) } ## @description Large display for the user console @@ -334,29 +453,6 @@ function big_console_header printf "\n\n" } -## @description Remove {color} tags from a string -## @audience public -## @stability stable -## @replaceable no -## @param string -## @return string -function colorstripper -{ - local string=$1 - shift 1 - - local green="" - local white="" - local red="" - local blue="" - - echo "${string}" | \ - ${SED} -e "s,{color:red},${red},g" \ - -e "s,{color:green},${green},g" \ - -e "s,{color:blue},${blue},g" \ - -e "s,{color},${white},g" -} - ## @description Find the largest size of a column of an array ## @audience private ## @stability evolving @@ -391,7 +487,7 @@ function find_java_home { start_clock if [[ -z ${JAVA_HOME:-} ]]; then - case $(uname -s) in + case ${OSTYPE} in Darwin) if [[ -z "${JAVA_HOME}" ]]; then if [[ -x /usr/libexec/java_home ]]; then @@ -409,7 +505,7 @@ function find_java_home if [[ -z ${JAVA_HOME:-} ]]; then echo "JAVA_HOME is not defined." - add_jira_table -1 pre-patch "JAVA_HOME is not defined." + add_vote_table -1 pre-patch "JAVA_HOME is not defined." return 1 fi return 0 @@ -420,26 +516,17 @@ function find_java_home ## @stability stable ## @audience public ## @returns ${JIRACLI} exit code -function write_to_jira +function write_comment { local -r commentfile=${1} shift - local retval + local retval=0 if [[ ${OFFLINE} == false && ${JENKINS} == true ]]; then - export USER=hudson - # shellcheck disable=SC2086 - ${JIRACLI} --comment "$(cat ${commentfile})" \ - -s https://issues.apache.org/jira \ - -a addcomment -u hadoopqa \ - -p "${JIRA_PASSWD}" \ - --issue "${ISSUE}" + ${BUGSYSTEM}_write_comment "${commentfile}" retval=$? - ${JIRACLI} -s https://issues.apache.org/jira \ - -a logout -u hadoopqa \ - -p "${JIRA_PASSWD}" fi return ${retval} } @@ -457,44 +544,46 @@ function verify_patchdir_still_exists local extra="" if [[ ! -d ${PATCH_DIR} ]]; then - rm "${commentfile}" 2>/dev/null - - echo "(!) The patch artifact directory has been removed! " > "${commentfile}" - echo "This is a fatal error for test-patch.sh. Aborting. " >> "${commentfile}" - echo - cat ${commentfile} - echo - if [[ ${JENKINS} == true ]]; then - if [[ -n ${NODE_NAME} ]]; then - extra=" (node ${NODE_NAME})" - fi - echo "Jenkins${extra} information at ${BUILD_URL} may provide some hints. " >> "${commentfile}" + rm "${commentfile}" 2>/dev/null - write_to_jira ${commentfile} + echo "(!) The patch artifact directory has been removed! " > "${commentfile}" + echo "This is a fatal error for test-patch.sh. Aborting. " >> "${commentfile}" + echo + cat ${commentfile} + echo + if [[ ${JENKINS} == true ]]; then + if [[ -n ${NODE_NAME} ]]; then + extra=" (node ${NODE_NAME})" fi + echo "Jenkins${extra} information at ${BUILD_URL} may provide some hints. " >> "${commentfile}" - rm "${commentfile}" - cleanup_and_exit ${RESULT} + write_comment ${commentfile} fi + + rm "${commentfile}" + cleanup_and_exit ${RESULT} + fi } -## @description generate a list of all files and line numbers that -## @description that were added/changed in the source repo +## @description generate a list of all files and line numbers in $GITDIFFLINES that +## @description that were added/changed in the source repo. $GITDIFFCONTENT +## @description is same file, but also includes the content of those lines ## @audience private ## @stability stable -## @params filename ## @replaceable no function compute_gitdiff { - local outfile=$1 local file local line local startline local counter local numlines local actual + local content + local outfile="${PATCH_DIR}/computegitdiff.${RANDOM}" pushd "${BASEDIR}" >/dev/null + ${GIT} add --all --intent-to-add while read line; do if [[ ${line} =~ ^\+\+\+ ]]; then file="./"$(echo "${line}" | cut -f2- -d/) @@ -511,11 +600,27 @@ function compute_gitdiff numlines=1 fi counter=0 - until [[ ${counter} -gt ${numlines} ]]; do + # it isn't obvious, but on MOST platforms under MOST use cases, + # this is faster than using sed, and definitely faster than using + # awk. + # http://unix.stackexchange.com/questions/47407/cat-line-x-to-line-y-on-a-huge-file + # has a good discussion w/benchmarks + # + # note that if tail is still sending data through the pipe, but head gets enough + # to do what was requested, head will exit, leaving tail with a broken pipe. + # we're going to send stderr to /dev/null and ignore the error since head's + # output is really what we're looking for + tail -n "+${startline}" "${file}" 2>/dev/null | head -n ${numlines} > "${outfile}" + oldifs=${IFS} + IFS='' + while read -r content; do ((actual=counter+startline)) - echo "${file}:${actual}:" >> "${outfile}" + echo "${file}:${actual}:" >> "${GITDIFFLINES}" + printf "%s:%s:%s\n" "${file}" "${actual}" "${content}" >> "${GITDIFFCONTENT}" ((counter=counter+1)) - done + done < "${outfile}" + rm "${outfile}" + IFS=${oldifs} fi done < <("${GIT}" diff --unified=0 --no-color) popd >/dev/null @@ -540,35 +645,39 @@ function echo_and_redirect verify_patchdir_still_exists find "${BASEDIR}" -type d -exec chmod +x {} \; + # to the screen echo "${*} > ${logfile} 2>&1" - "${@}" > "${logfile}" 2>&1 + # to the log + echo "${*}" > "${logfile}" + # the actual command + "${@}" >> "${logfile}" 2>&1 } -## @description is PATCH_DIR relative to BASEDIR? +## @description is a given directory relative to BASEDIR? ## @audience public ## @stability stable ## @replaceable yes -## @returns 1 - no, PATCH_DIR -## @returns 0 - yes, PATCH_DIR - BASEDIR -function relative_patchdir +## @param path +## @returns 1 - no, path +## @returns 0 - yes, path - BASEDIR +function relative_dir { - local p=${PATCH_DIR#${BASEDIR}} + local p=${1#${BASEDIR}} - if [[ ${#p} -eq ${#PATCH_DIR} ]]; then - echo ${p} + if [[ ${#p} -eq ${#1} ]]; then + echo "${p}" return 1 fi p=${p#/} - echo ${p} + echo "${p}" return 0 } - ## @description Print the usage information ## @audience public ## @stability stable ## @replaceable no -function hadoop_usage +function testpatch_usage { local -r up=$(echo ${PROJECT_NAME} | tr '[:lower:]' '[:upper:]') @@ -582,25 +691,36 @@ function hadoop_usage echo "Options:" echo "--basedir=<dir> The directory to apply the patch to (default current directory)" echo "--branch=<ref> Forcibly set the branch" - echo "--branch-default=<ref> If the branch isn't forced and we don't detect one in the patch name, use this branch (default 'trunk')" + echo "--branch-default=<ref> If the branch isn't forced and we don't detect one in the patch name, use this branch (default 'master')" + #not quite working yet + #echo "--bugsystem=<type> The bug system in use ('jira', the default, or 'github')" echo "--build-native=<bool> If true, then build native components (default 'true')" - echo "--contrib-guide=<url> URL to point new users towards project conventions. (default Hadoop's wiki)" + echo "--build-tool=<tool> Pick which build tool to focus around (maven, ant)" + echo "--contrib-guide=<url> URL to point new users towards project conventions. (default: ${HOW_TO_CONTRIBUTE} )" echo "--debug If set, then output some extra stuff to stderr" echo "--dirty-workspace Allow the local git workspace to have uncommitted changes" - echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)" - echo "--findbugs-strict-precheck If there are Findbugs warnings during precheck, fail" - echo "--issue-re=<expr> Bash regular expression to use when trying to find a jira ref in the patch name (default '^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$')" + echo "--docker Spawn a docker container" + echo "--dockerfile=<file> Dockerfile fragment to use as the base" + echo "--issue-re=<expr> Bash regular expression to use when trying to find a jira ref in the patch name (default: \'${ISSUE_RE}\')" + echo "--java-home=<path> Set JAVA_HOME (In Docker mode, this should be local to the image)" + echo "--multijdkdirs=<paths> Comma delimited lists of JDK paths to use for multi-JDK tests" + echo "--multijdktests=<list> Comma delimited tests to use when multijdkdirs is used. (default: javac,javadoc,unit)" echo "--modulelist=<list> Specify additional modules to test (comma delimited)" echo "--offline Avoid connecting to the Internet" - echo "--patch-dir=<dir> The directory for working and output files (default '/tmp/${PROJECT_NAME}-test-patch/pid')" + echo "--patch-dir=<dir> The directory for working and output files (default '/tmp/test-patch-${PROJECT_NAME}/pid')" + echo "--personality=<file> The personality file to load" echo "--plugins=<dir> A directory of user provided plugins. see test-patch.d for examples (default empty)" - echo "--project=<name> The short name for project currently using test-patch (default 'hadoop')" + echo "--project=<name> The short name for project currently using test-patch (default 'yetus')" echo "--resetrepo Forcibly clean the repo" echo "--run-tests Run all relevant tests below the base directory" echo "--skip-system-plugins Do not load plugins from ${BINDIR}/test-patch.d" + echo "--summarize=<bool> Allow tests to summarize results" echo "--testlist=<list> Specify which subsystem tests to use (comma delimited)" - + echo "--test-parallel=<bool> Run multiple tests in parallel (default false in developer mode, true in Jenkins mode)" + echo "--test-threads=<int> Number of tests to run in parallel (default defined in ${PROJECT_NAME} build)" + echo "" echo "Shell binary overrides:" + echo "--ant-cmd=<cmd> The 'ant' command to use (default \${ANT_HOME}/bin/ant, or 'ant')" echo "--awk-cmd=<cmd> The 'awk' command to use (default 'awk')" echo "--diff-cmd=<cmd> The GNU-compatible 'diff' command to use (default 'diff')" echo "--file-cmd=<cmd> The 'file' command to use (default 'file')" @@ -608,17 +728,24 @@ function hadoop_usage echo "--grep-cmd=<cmd> The 'grep' command to use (default 'grep')" echo "--mvn-cmd=<cmd> The 'mvn' command to use (default \${MAVEN_HOME}/bin/mvn, or 'mvn')" echo "--patch-cmd=<cmd> The 'patch' command to use (default 'patch')" - echo "--ps-cmd=<cmd> The 'ps' command to use (default 'ps')" echo "--sed-cmd=<cmd> The 'sed' command to use (default 'sed')" echo echo "Jenkins-only options:" echo "--jenkins Run by Jenkins (runs tests and posts results to JIRA)" + echo "--build-url Set the build location web page" echo "--eclipse-home=<path> Eclipse home directory (default ECLIPSE_HOME environment variable)" - echo "--jira-cmd=<cmd> The 'jira' command to use (default 'jira')" - echo "--jira-password=<pw> The password for the 'jira' command" echo "--mv-patch-dir Move the patch-dir into the basedir during cleanup." echo "--wget-cmd=<cmd> The 'wget' command to use (default 'wget')" + + importplugins + + for plugin in ${PLUGINS} ${BUGSYSTEMS}; do + if declare -f ${plugin}_usage >/dev/null 2>&1; then + echo + "${plugin}_usage" + fi + done } ## @description Interpret the command line parameters @@ -634,6 +761,9 @@ function parse_args for i in "$@"; do case ${i} in + --ant-cmd=*) + ANT=${i#*=} + ;; --awk-cmd=*) AWK=${i#*=} ;; @@ -646,14 +776,23 @@ function parse_args --branch-default=*) PATCH_BRANCH_DEFAULT=${i#*=} ;; + --bugsystem=*) + BUGSYSTEM=${i#*=} + ;; --build-native=*) BUILD_NATIVE=${i#*=} ;; + --build-tool=*) + BUILDTOOL=${i#*=} + ;; + --build-url=*) + BUILD_URL=${i#*=} + ;; --contrib-guide=*) HOW_TO_CONTRIBUTE=${i#*=} ;; --debug) - HADOOP_SHELL_SCRIPT_DEBUG=true + TP_SHELL_SCRIPT_DEBUG=true ;; --diff-cmd=*) DIFF=${i#*=} @@ -661,18 +800,21 @@ function parse_args --dirty-workspace) DIRTY_WORKSPACE=true ;; + --docker) + DOCKERSUPPORT=true + ;; + --dockerfile=*) + DOCKERFILE=${i#*=} + ;; + --dockermode) + DOCKERMODE=true + ;; --eclipse-home=*) ECLIPSE_HOME=${i#*=} ;; --file-cmd=*) FILE=${i#*=} ;; - --findbugs-home=*) - FINDBUGS_HOME=${i#*=} - ;; - --findbugs-strict-precheck) - FINDBUGS_WARNINGS_FAIL_PRECHECK=true - ;; --git-cmd=*) GIT=${i#*=} ;; @@ -680,7 +822,7 @@ function parse_args GREP=${i#*=} ;; --help|-help|-h|help|--h|--\?|-\?|\?) - hadoop_usage + testpatch_usage exit 0 ;; --issue-re=*) @@ -691,17 +833,22 @@ function parse_args ;; --jenkins) JENKINS=true - ;; - --jira-cmd=*) - JIRACLI=${i#*=} - ;; - --jira-password=*) - JIRA_PASSWD=${i#*=} + TEST_PARALLEL=${TEST_PARALLEL:-true} ;; --modulelist=*) USER_MODULE_LIST=${i#*=} USER_MODULE_LIST=${USER_MODULE_LIST//,/ } - hadoop_debug "Manually forcing modules ${USER_MODULE_LIST}" + yetus_debug "Manually forcing modules ${USER_MODULE_LIST}" + ;; + --multijdkdirs=*) + JDK_DIR_LIST=${i#*=} + JDK_DIR_LIST=${JDK_DIR_LIST//,/ } + yetus_debug "Multi-JVM mode activated with ${JDK_DIR_LIST}" + ;; + --multijdktests=*) + JDK_TEST_LIST=${i#*=} + JDK_TEST_LIST=${JDK_TEST_LIST//,/ } + yetus_debug "Multi-JVM test list: ${JDK_TEST_LIST}" ;; --mvn-cmd=*) MVN=${i#*=} @@ -718,19 +865,17 @@ function parse_args --patch-dir=*) USER_PATCH_DIR=${i#*=} ;; + --personality=*) + PERSONALITY=${i#*=} + ;; --plugins=*) USER_PLUGIN_DIR=${i#*=} ;; --project=*) PROJECT_NAME=${i#*=} ;; - --ps-cmd=*) - PS=${i#*=} - ;; --reexec) REEXECED=true - start_clock - add_jira_table 0 reexec "dev-support patch detected." ;; --resetrepo) RESETREPO=true @@ -741,62 +886,81 @@ function parse_args --skip-system-plugins) LOAD_SYSTEM_PLUGINS=false ;; + --summarize=*) + ALLOWSUMMARIES=${i#*=} + ;; --testlist=*) testlist=${i#*=} testlist=${testlist//,/ } for j in ${testlist}; do - hadoop_debug "Manually adding patch test subsystem ${j}" + yetus_debug "Manually adding patch test subsystem ${j}" add_test "${j}" done ;; + --test-parallel=*) + TEST_PARALLEL=${i#*=} + ;; + --test-threads=*) + # shellcheck disable=SC2034 + TEST_THREADS=${i#*=} + ;; + --tpglobaltimer=*) + GLOBALTIMER=${i#*=} + ;; + --tpreexectimer=*) + REEXECLAUNCHTIMER=${i#*=} + ;; --wget-cmd=*) WGET=${i#*=} ;; + --*) + ## PATCH_OR_ISSUE can't be a --. So this is probably + ## a plugin thing. + continue + ;; *) PATCH_OR_ISSUE=${i} ;; esac done - # if we requested offline, pass that to mvn - if [[ ${OFFLINE} == "true" ]] ; then - MAVEN_ARGS=(${MAVEN_ARGS[@]} --offline) + if [[ -n ${REEXECLAUNCHTIMER} ]]; then + TIMER=${REEXECLAUNCHTIMER}; + else + start_clock fi - # we need absolute dir for ${BASEDIR} - cd "${CWD}" - BASEDIR=$(cd -P -- "${BASEDIR}" >/dev/null && pwd -P) + if [[ ${REEXECED} == true + && ${DOCKERMODE} == true ]]; then + add_vote_table 0 reexec "docker + precommit patch detected." + elif [[ ${REEXECED} == true ]]; then + add_vote_table 0 reexec "precommit patch detected." + elif [[ ${DOCKERMODE} == true ]]; then + add_vote_table 0 reexec "docker mode." + fi - if [[ ${BUILD_NATIVE} == "true" ]] ; then - NATIVE_PROFILE=-Pnative - REQUIRE_TEST_LIB_HADOOP=-Drequire.test.libhadoop + # if we requested offline, pass that to mvn + if [[ ${OFFLINE} == "true" ]]; then + MAVEN_ARGS=(${MAVEN_ARGS[@]} --offline) + ANT_ARGS=(${ANT_ARGS[@]} -Doffline=) fi + if [[ -z "${PATCH_OR_ISSUE}" ]]; then - hadoop_usage + testpatch_usage exit 1 fi - if [[ ${JENKINS} == "true" ]] ; then - echo "Running in Jenkins mode" - ISSUE=${PATCH_OR_ISSUE} - RESETREPO=true - # shellcheck disable=SC2034 - ECLIPSE_PROPERTY="-Declipse.home=${ECLIPSE_HOME}" - else - if [[ ${RESETREPO} == "true" ]] ; then - echo "Running in destructive (--resetrepo) developer mode" - else - echo "Running in developer mode" - fi - JENKINS=false - fi + + # we need absolute dir for ${BASEDIR} + cd "${STARTINGDIR}" + BASEDIR=$(cd -P -- "${BASEDIR}" >/dev/null && pwd -P) if [[ -n ${USER_PATCH_DIR} ]]; then PATCH_DIR="${USER_PATCH_DIR}" else - PATCH_DIR=/tmp/${PROJECT_NAME}-test-patch/$$ + PATCH_DIR=/tmp/test-patch-${PROJECT_NAME}/$$ fi - cd "${CWD}" + cd "${STARTINGDIR}" if [[ ! -d ${PATCH_DIR} ]]; then mkdir -p "${PATCH_DIR}" if [[ $? == 0 ]] ; then @@ -810,26 +974,76 @@ function parse_args # we need absolute dir for PATCH_DIR PATCH_DIR=$(cd -P -- "${PATCH_DIR}" >/dev/null && pwd -P) - GITDIFFLINES=${PATCH_DIR}/gitdifflines.txt + if [[ ${JENKINS} == "true" ]]; then + echo "Running in Jenkins mode" + ISSUE=${PATCH_OR_ISSUE} + RESETREPO=true + # shellcheck disable=SC2034 + ECLIPSE_PROPERTY="-Declipse.home=${ECLIPSE_HOME}" + else + if [[ ${RESETREPO} == "true" ]] ; then + echo "Running in destructive (--resetrepo) developer mode" + else + echo "Running in developer mode" + fi + JENKINS=false + fi + + if [[ -n "${USER_PLUGIN_DIR}" ]]; then + USER_PLUGIN_DIR=$(cd -P -- "${USER_PLUGIN_DIR}" >/dev/null && pwd -P) + fi + + GITDIFFLINES="${PATCH_DIR}/gitdifflines.txt" + GITDIFFCONTENT="${PATCH_DIR}/gitdiffcontent.txt" } ## @description Locate the pom.xml file for a given directory ## @audience private ## @stability stable ## @replaceable no -## @return directory containing the pom.xml -function find_pom_dir +## @return directory containing the pom.xml. Nothing returned if not found. +function find_pomxml_dir { local dir dir=$(dirname "$1") - hadoop_debug "Find pom dir for: ${dir}" + yetus_debug "Find pom.xml dir for: ${dir}" while builtin true; do if [[ -f "${dir}/pom.xml" ]];then echo "${dir}" - hadoop_debug "Found: ${dir}" + yetus_debug "Found: ${dir}" + return + elif [[ ${dir} == "." ]]; then + yetus_error "ERROR: pom.xml is not found. Make sure the target is a Maven-based project." + return + else + dir=$(dirname "${dir}") + fi + done +} + +## @description Locate the build.xml file for a given directory +## @audience private +## @stability stable +## @replaceable no +## @return directory containing the build.xml. Nothing returned if not found. +function find_buildxml_dir +{ + local dir + + dir=$(dirname "$1") + + yetus_debug "Find build.xml dir for: ${dir}" + + while builtin true; do + if [[ -f "${dir}/build.xml" ]];then + echo "${dir}" + yetus_debug "Found: ${dir}" + return + elif [[ ${dir} == "." ]]; then + yetus_error "ERROR: build.xml is not found. Make sure the target is a Ant-based project." return else dir=$(dirname "${dir}") @@ -858,33 +1072,65 @@ function find_changed_files | sort -u) } -## @description Find the modules of the maven build that ${PATCH_DIR}/patch modifies +## @description Find the modules of the build that ${PATCH_DIR}/patch modifies ## @audience private ## @stability stable ## @replaceable no -## @return None; sets ${CHANGED_MODULES} +## @return None; sets ${CHANGED_MODULES} and ${CHANGED_UNFILTERED_MODULES} function find_changed_modules { # Come up with a list of changed files into ${TMP} local pomdirs + local pomdir local module local pommods # Now find all the modules that were changed for file in ${CHANGED_FILES}; do - #shellcheck disable=SC2086 - pomdirs="${pomdirs} $(find_pom_dir ${file})" + case ${BUILDTOOL} in + maven) + #shellcheck disable=SC2086 + pomdir=$(find_pomxml_dir ${file}) + if [[ -z ${pomdir} ]]; then + output_to_console 1 + output_to_bugsystem 1 + cleanup_and_exit 1 + fi + pomdirs="${pomdirs} ${pomdir}" + ;; + ant) + #shellcheck disable=SC2086 + pomdir=$(find_buildxml_dir ${file}) + if [[ -z ${pomdir} ]]; then + output_to_console 1 + output_to_bugsystem 1 + cleanup_and_exit 1 + fi + pomdirs="${pomdirs} ${pomdir}" + ;; + *) + yetus_error "ERROR: Unsupported build tool." + output_to_console 1 + output_to_bugsystem 1 + cleanup_and_exit 1 + ;; + esac done - # Filter out modules without code - for module in ${pomdirs}; do - ${GREP} "<packaging>pom</packaging>" "${module}/pom.xml" > /dev/null - if [[ "$?" != 0 ]]; then - pommods="${pommods} ${module}" - fi - done + #shellcheck disable=SC2086,SC2034 + CHANGED_UNFILTERED_MODULES=$(echo ${pomdirs} ${USER_MODULE_LIST} | tr ' ' '\n' | sort -u) + + if [[ ${BUILDTOOL} == maven ]]; then + # Filter out modules without code + for module in ${pomdirs}; do + ${GREP} "<packaging>pom</packaging>" "${module}/pom.xml" > /dev/null + if [[ "$?" != 0 ]]; then + pommods="${pommods} ${module}" + fi + done + fi - #shellcheck disable=SC2086 + #shellcheck disable=SC2086,SC2034 CHANGED_MODULES=$(echo ${pommods} ${USER_MODULE_LIST} | tr ' ' '\n' | sort -u) } @@ -904,38 +1150,38 @@ function git_checkout cd "${BASEDIR}" if [[ ! -d .git ]]; then - hadoop_error "ERROR: ${BASEDIR} is not a git repo." + yetus_error "ERROR: ${BASEDIR} is not a git repo." cleanup_and_exit 1 fi if [[ ${RESETREPO} == "true" ]] ; then ${GIT} reset --hard if [[ $? != 0 ]]; then - hadoop_error "ERROR: git reset is failing" + yetus_error "ERROR: git reset is failing" cleanup_and_exit 1 fi # if PATCH_DIR is in BASEDIR, then we don't want # git wiping it out. - exemptdir=$(relative_patchdir) + exemptdir=$(relative_dir "${PATCH_DIR}") if [[ $? == 1 ]]; then ${GIT} clean -xdf else # we do, however, want it emptied of all _files_. # we need to leave _directories_ in case we are in # re-exec mode (which places a directory full of stuff in it) - hadoop_debug "Exempting ${exemptdir} from clean" + yetus_debug "Exempting ${exemptdir} from clean" rm "${PATCH_DIR}/*" 2>/dev/null ${GIT} clean -xdf -e "${exemptdir}" fi if [[ $? != 0 ]]; then - hadoop_error "ERROR: git clean is failing" + yetus_error "ERROR: git clean is failing" cleanup_and_exit 1 fi ${GIT} checkout --force "${PATCH_BRANCH_DEFAULT}" if [[ $? != 0 ]]; then - hadoop_error "ERROR: git checkout --force ${PATCH_BRANCH_DEFAULT} is failing" + yetus_error "ERROR: git checkout --force ${PATCH_BRANCH_DEFAULT} is failing" cleanup_and_exit 1 fi @@ -949,14 +1195,14 @@ function git_checkout if [[ ${OFFLINE} == false ]]; then ${GIT} pull --rebase if [[ $? != 0 ]]; then - hadoop_error "ERROR: git pull is failing" + yetus_error "ERROR: git pull is failing" cleanup_and_exit 1 fi fi # forcibly checkout this branch or git ref ${GIT} checkout --force "${PATCH_BRANCH}" if [[ $? != 0 ]]; then - hadoop_error "ERROR: git checkout ${PATCH_BRANCH} is failing" + yetus_error "ERROR: git checkout ${PATCH_BRANCH} is failing" cleanup_and_exit 1 fi @@ -965,7 +1211,7 @@ function git_checkout if [[ ${OFFLINE} == false ]]; then ${GIT} pull --rebase if [[ $? != 0 ]]; then - hadoop_error "ERROR: git pull is failing" + yetus_error "ERROR: git pull is failing" cleanup_and_exit 1 fi fi @@ -974,9 +1220,9 @@ function git_checkout status=$(${GIT} status --porcelain) if [[ "${status}" != "" && -z ${DIRTY_WORKSPACE} ]] ; then - hadoop_error "ERROR: --dirty-workspace option not provided." - hadoop_error "ERROR: can't run in a workspace that contains the following modifications" - hadoop_error "${status}" + yetus_error "ERROR: --dirty-workspace option not provided." + yetus_error "ERROR: can't run in a workspace that contains the following modifications" + yetus_error "${status}" cleanup_and_exit 1 fi @@ -1005,114 +1251,42 @@ function git_checkout echo "Testing ${ISSUE} patch on ${PATCH_BRANCH}." fi - add_jira_footer "git revision" "${PATCH_BRANCH} / ${GIT_REVISION}" + add_footer_table "git revision" "${PATCH_BRANCH} / ${GIT_REVISION}" - if [[ ! -f ${BASEDIR}/pom.xml ]]; then - hadoop_error "ERROR: This verison of test-patch.sh only supports Maven-based builds. Aborting." - add_jira_table -1 pre-patch "Unsupported build system." - output_to_jira 1 - cleanup_and_exit 1 - fi return 0 } -## @description Confirm the source environment is compilable +## @description Confirm the given branch is a member of the list of space +## @description delimited branches or a git ref ## @audience private -## @stability stable +## @stability evolving ## @replaceable no +## @param branch +## @param branchlist ## @return 0 on success ## @return 1 on failure -function precheck_without_patch +function verify_valid_branch { - local -r mypwd=$(pwd) + local branches=$1 + local check=$2 + local i - big_console_header "Pre-patch ${PATCH_BRANCH} Java verification" + # shortcut some common + # non-resolvable names + if [[ -z ${check} ]]; then + return 1 + fi - start_clock + if [[ ${check} == patch ]]; then + return 1 + fi - verify_needed_test javac + if [[ ${check} =~ ^git ]]; then + ref=$(echo "${check}" | cut -f2 -dt) + count=$(echo "${ref}" | wc -c | tr -d ' ') - if [[ $? == 1 ]]; then - echo "Compiling ${mypwd}" - echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavacWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch - if [[ $? != 0 ]] ; then - echo "${PATCH_BRANCH} compilation is broken?" - add_jira_table -1 pre-patch "${PATCH_BRANCH} compilation may be broken." - return 1 - fi - else - echo "Patch does not appear to need javac tests." - fi - - verify_needed_test javadoc - - if [[ $? == 1 ]]; then - echo "Javadoc'ing ${mypwd}" - echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess - if [[ $? != 0 ]] ; then - echo "Pre-patch ${PATCH_BRANCH} javadoc compilation is broken?" - add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} JavaDoc compilation may be broken." - return 1 - fi - else - echo "Patch does not appear to need javadoc tests." - fi - - verify_needed_test site - - if [[ $? == 1 ]]; then - echo "site creation for ${mypwd}" - echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}SiteWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess - if [[ $? != 0 ]] ; then - echo "Pre-patch ${PATCH_BRANCH} site compilation is broken?" - add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} site compilation may be broken." - return 1 - fi - else - echo "Patch does not appear to need site tests." - fi - - precheck_findbugs - - if [[ $? != 0 ]] ; then - return 1 - fi - - add_jira_table 0 pre-patch "Pre-patch ${PATCH_BRANCH} compilation is healthy." - return 0 -} - -## @description Confirm the given branch is a member of the list of space -## @description delimited branches or a git ref -## @audience private -## @stability evolving -## @replaceable no -## @param branch -## @param branchlist -## @return 0 on success -## @return 1 on failure -function verify_valid_branch -{ - local branches=$1 - local check=$2 - local i - - # shortcut some common - # non-resolvable names - if [[ -z ${check} ]]; then - return 1 - fi - - if [[ ${check} == patch ]]; then - return 1 - fi - - if [[ ${check} =~ ^git ]]; then - ref=$(echo "${check}" | cut -f2 -dt) - count=$(echo "${ref}" | wc -c | tr -d ' ') - - if [[ ${count} == 8 || ${count} == 41 ]]; then - return 0 + if [[ ${count} == 8 || ${count} == 41 ]]; then + return 0 fi return 1 fi @@ -1136,7 +1310,7 @@ function determine_branch local allbranches local patchnamechunk - hadoop_debug "Determine branch" + yetus_debug "Determine branch" # something has already set this, so move on if [[ -n ${PATCH_BRANCH} ]]; then @@ -1155,12 +1329,12 @@ function determine_branch allbranches=$(${GIT} branch -r | tr -d ' ' | ${SED} -e s,origin/,,g) for j in "${PATCHURL}" "${PATCH_OR_ISSUE}"; do - hadoop_debug "Determine branch: starting with ${j}" + yetus_debug "Determine branch: starting with ${j}" # shellcheck disable=SC2016 patchnamechunk=$(echo "${j}" | ${AWK} -F/ '{print $NF}') # ISSUE.branch.##.patch - hadoop_debug "Determine branch: ISSUE.branch.##.patch" + yetus_debug "Determine branch: ISSUE.branch.##.patch" PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f2 -d. ) verify_valid_branch "${allbranches}" "${PATCH_BRANCH}" if [[ $? == 0 ]]; then @@ -1168,7 +1342,7 @@ function determine_branch fi # ISSUE-branch-##.patch - hadoop_debug "Determine branch: ISSUE-branch-##.patch" + yetus_debug "Determine branch: ISSUE-branch-##.patch" PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1,2 -d-) verify_valid_branch "${allbranches}" "${PATCH_BRANCH}" if [[ $? == 0 ]]; then @@ -1176,7 +1350,7 @@ function determine_branch fi # ISSUE-##.patch.branch - hadoop_debug "Determine branch: ISSUE-##.patch.branch" + yetus_debug "Determine branch: ISSUE-##.patch.branch" # shellcheck disable=SC2016 PATCH_BRANCH=$(echo "${patchnamechunk}" | ${AWK} -F. '{print $NF}') verify_valid_branch "${allbranches}" "${PATCH_BRANCH}" @@ -1185,7 +1359,7 @@ function determine_branch fi # ISSUE-branch.##.patch - hadoop_debug "Determine branch: ISSUE-branch.##.patch" + yetus_debug "Determine branch: ISSUE-branch.##.patch" # shellcheck disable=SC2016 PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | ${AWK} -F. '{print $(NF-2)}' 2>/dev/null) verify_valid_branch "${allbranches}" "${PATCH_BRANCH}" @@ -1210,7 +1384,7 @@ function determine_issue local patchnamechunk local maybeissue - hadoop_debug "Determine issue" + yetus_debug "Determine issue" # we can shortcut jenkins if [[ ${JENKINS} == true ]]; then @@ -1241,13 +1415,13 @@ function add_test { local testname=$1 - hadoop_debug "Testing against ${testname}" + yetus_debug "Testing against ${testname}" if [[ -z ${NEEDED_TESTS} ]]; then - hadoop_debug "Setting tests to ${testname}" + yetus_debug "Setting tests to ${testname}" NEEDED_TESTS=${testname} elif [[ ! ${NEEDED_TESTS} =~ ${testname} ]] ; then - hadoop_debug "Adding ${testname}" + yetus_debug "Adding ${testname}" NEEDED_TESTS="${NEEDED_TESTS} ${testname}" fi } @@ -1279,44 +1453,8 @@ function determine_needed_tests local i for i in ${CHANGED_FILES}; do - if [[ ${i} =~ src/main/webapp ]]; then - hadoop_debug "tests/webapp: ${i}" - elif [[ ${i} =~ \.sh - || ${i} =~ \.cmd - ]]; then - hadoop_debug "tests/shell: ${i}" - elif [[ ${i} =~ \.md$ - || ${i} =~ \.md\.vm$ - || ${i} =~ src/site - || ${i} =~ src/main/docs - ]]; then - hadoop_debug "tests/site: ${i}" - add_test site - elif [[ ${i} =~ \.c$ - || ${i} =~ \.cc$ - || ${i} =~ \.h$ - || ${i} =~ \.hh$ - || ${i} =~ \.proto$ - || ${i} =~ src/test - || ${i} =~ \.cmake$ - || ${i} =~ CMakeLists.txt - ]]; then - hadoop_debug "tests/units: ${i}" - add_test javac - add_test unit - elif [[ ${i} =~ pom.xml$ - || ${i} =~ \.java$ - || ${i} =~ src/main - ]]; then - hadoop_debug "tests/javadoc+units: ${i}" - add_test javadoc - add_test javac - add_test unit - fi - if [[ ${i} =~ \.java$ ]]; then - add_test findbugs - fi + personality_file_tests "${i}" for plugin in ${PLUGINS}; do if declare -f ${plugin}_filefilter >/dev/null 2>&1; then @@ -1325,7 +1463,7 @@ function determine_needed_tests done done - add_jira_footer "Optional Tests" "${NEEDED_TESTS}" + add_footer_table "Optional Tests" "${NEEDED_TESTS}" } ## @description Given ${PATCH_ISSUE}, determine what type of patch file is in use, and do the @@ -1338,7 +1476,7 @@ function determine_needed_tests function locate_patch { local notSureIfPatch=false - hadoop_debug "locate patch" + yetus_debug "locate patch" if [[ -f ${PATCH_OR_ISSUE} ]]; then PATCH_FILE="${PATCH_OR_ISSUE}" @@ -1349,36 +1487,64 @@ function locate_patch else ${WGET} -q -O "${PATCH_DIR}/jira" "http://issues.apache.org/jira/browse/${PATCH_OR_ISSUE}"; - if [[ $? != 0 ]];then - hadoop_error "ERROR: Unable to determine what ${PATCH_OR_ISSUE} may reference." - cleanup_and_exit 1 - fi - - if [[ $(${GREP} -c 'Patch Available' "${PATCH_DIR}/jira") == 0 ]] ; then - if [[ ${JENKINS} == true ]]; then - hadoop_error "ERROR: ${PATCH_OR_ISSUE} is not \"Patch Available\"." + case $? in + 0) + ;; + 2) + yetus_error "ERROR: .wgetrc/.netrc parsing error." cleanup_and_exit 1 - else - hadoop_error "WARNING: ${PATCH_OR_ISSUE} is not \"Patch Available\"." + ;; + 3) + yetus_error "ERROR: File IO error." + cleanup_and_exit 1 + ;; + 4) + yetus_error "ERROR: URL ${PATCH_OR_ISSUE} is unreachable." + cleanup_and_exit 1 + ;; + *) + # we want to try and do as much as we can in docker mode, + # but if the patch was passed as a file, then we may not + # be able to continue. + if [[ ${REEXECED} == true + && -f "${PATCH_DIR}/patch" ]]; then + PATCH_FILE="${PATCH_DIR}/patch" + else + yetus_error "ERROR: Unable to fetch ${PATCH_OR_ISSUE}." + cleanup_and_exit 1 + fi + ;; + esac + + if [[ -z "${PATCH_FILE}" ]]; then + if [[ $(${GREP} -c 'Patch Available' "${PATCH_DIR}/jira") == 0 ]] ; then + if [[ ${JENKINS} == true ]]; then + yetus_error "ERROR: ${PATCH_OR_ISSUE} is not \"Patch Available\"." + cleanup_and_exit 1 + else + yetus_error "WARNING: ${PATCH_OR_ISSUE} is not \"Patch Available\"." + fi fi - fi - relativePatchURL=$(${GREP} -o '"/jira/secure/attachment/[0-9]*/[^"]*' "${PATCH_DIR}/jira" | ${GREP} -v -e 'htm[l]*$' | sort | tail -1 | ${GREP} -o '/jira/secure/attachment/[0-9]*/[^"]*') - PATCHURL="http://issues.apache.org${relativePatchURL}"; - if [[ ! ${PATCHURL} =~ \.patch$ ]]; then - notSureIfPatch=true + relativePatchURL=$(${GREP} -o '"/jira/secure/attachment/[0-9]*/[^"]*' "${PATCH_DIR}/jira" | ${GREP} -v -e 'htm[l]*$' | sort | tail -1 | ${GREP} -o '/jira/secure/attachment/[0-9]*/[^"]*') + PATCHURL="http://issues.apache.org${relativePatchURL}"; + if [[ ! ${PATCHURL} =~ \.patch$ ]]; then + notSureIfPatch=true + fi + patchNum=$(echo "${PATCHURL}" | ${GREP} -o '[0-9]*/' | ${GREP} -o '[0-9]*') + echo "${ISSUE} patch is being downloaded at $(date) from" fi - patchNum=$(echo "${PATCHURL}" | ${GREP} -o '[0-9]*/' | ${GREP} -o '[0-9]*') - echo "${ISSUE} patch is being downloaded at $(date) from" fi - echo "${PATCHURL}" - add_jira_footer "Patch URL" "${PATCHURL}" - ${WGET} -q -O "${PATCH_DIR}/patch" "${PATCHURL}" - if [[ $? != 0 ]];then - hadoop_error "ERROR: ${PATCH_OR_ISSUE} could not be downloaded." - cleanup_and_exit 1 + if [[ -z "${PATCH_FILE}" ]]; then + echo "${PATCHURL}" + add_footer_table "Patch URL" "${PATCHURL}" + ${WGET} -q -O "${PATCH_DIR}/patch" "${PATCHURL}" + if [[ $? != 0 ]];then + yetus_error "ERROR: ${PATCH_OR_ISSUE} could not be downloaded." + cleanup_and_exit 1 + fi + PATCH_FILE="${PATCH_DIR}/patch" fi - PATCH_FILE="${PATCH_DIR}/patch" fi if [[ ! -f "${PATCH_DIR}/patch" ]]; then @@ -1386,18 +1552,19 @@ function locate_patch if [[ $? == 0 ]] ; then echo "Patch file ${PATCH_FILE} copied to ${PATCH_DIR}" else - hadoop_error "ERROR: Could not copy ${PATCH_FILE} to ${PATCH_DIR}" + yetus_error "ERROR: Could not copy ${PATCH_FILE} to ${PATCH_DIR}" cleanup_and_exit 1 fi fi + if [[ ${notSureIfPatch} == "true" ]]; then guess_patch_file "${PATCH_DIR}/patch" if [[ $? != 0 ]]; then - hadoop_error "ERROR: ${PATCHURL} is not a patch file." + yetus_error "ERROR: ${PATCHURL} is not a patch file." cleanup_and_exit 1 else - hadoop_debug "The patch ${PATCHURL} was not named properly, but it looks like a patch file. proceeding, but issue/branch matching might go awry." - add_jira_table 0 patch "The patch file was not named according to ${PROJECT_NAME}'s naming conventions. Please see ${HOW_TO_CONTRIBUTE} for instructions." + yetus_debug "The patch ${PATCHURL} was not named properly, but it looks like a patch file. proceeding, but issue/branch matching might go awry." + add_vote_table 0 patch "The patch file was not named according to ${PROJECT_NAME}'s naming conventions. Please see ${HOW_TO_CONTRIBUTE} for instructions." fi fi } @@ -1413,15 +1580,15 @@ function guess_patch_file local patch=$1 local fileOutput - hadoop_debug "Trying to guess is ${patch} is a patch file." + yetus_debug "Trying to guess is ${patch} is a patch file." fileOutput=$("${FILE}" "${patch}") if [[ $fileOutput =~ \ diff\ ]]; then - hadoop_debug "file magic says it's a diff." + yetus_debug "file magic says it's a diff." return 0 fi - fileOutput=$(head -n 1 "${patch}" | "${EGREP}" "^(From [a-z0-9]* Mon Sep 17 00:00:00 2001)|(diff .*)|(Index: .*)$") + fileOutput=$(head -n 1 "${patch}" | "${GREP}" -E "^(From [a-z0-9]* Mon Sep 17 00:00:00 2001)|(diff .*)|(Index: .*)$") if [[ $? == 0 ]]; then - hadoop_debug "first line looks like a patch file." + yetus_debug "first line looks like a patch file." return 0 fi return 1 @@ -1442,7 +1609,7 @@ function verify_patch_file "${BINDIR}/smart-apply-patch.sh" "${PATCH_DIR}/patch" dryrun if [[ $? != 0 ]] ; then echo "PATCH APPLICATION FAILED" - add_jira_table -1 patch "The patch command could not apply the patch during dryrun." + add_vote_table -1 patch "The patch command could not apply the patch during dryrun." return 1 else return 0 @@ -1464,17 +1631,91 @@ function apply_patch_file if [[ $? != 0 ]] ; then echo "PATCH APPLICATION FAILED" ((RESULT = RESULT + 1)) - add_jira_table -1 patch "The patch command could not apply the patch." + add_vote_table -1 patch "The patch command could not apply the patch." output_to_console 1 - output_to_jira 1 + output_to_bugsystem 1 cleanup_and_exit 1 fi return 0 } +## @description copy the test-patch binary bits to a new working dir, +## @description setting USER_PLUGIN_DIR and PERSONALITY to the new +## @description locations. +## @description this is used for test-patch in docker and reexec mode +## @audience private +## @stability evolving +## @replaceable no +function copytpbits +{ + local dockerdir + local dockfile + local person + # we need to copy/consolidate all the bits that might have changed + # that are considered part of test-patch. This *might* break + # things that do off-path includes, but there isn't much we can + # do about that, I don't think. + + # if we've already copied, then don't bother doing it again + if [[ ${STARTDIR} == ${PATCH_DIR}/precommit ]]; then + hadoop_debug "Skipping copytpbits; already copied once" + return + fi + + pushd "${STARTINGDIR}" >/dev/null + mkdir -p "${PATCH_DIR}/precommit/user-plugins" + mkdir -p "${PATCH_DIR}/precommit/personality" + mkdir -p "${PATCH_DIR}/precommit/test-patch-docker" -## @description If this actually patches the files used for the QA process -## @description under dev-support and its subdirectories, then + # copy our entire universe, preserving links, etc. + (cd "${BINDIR}"; tar cpf - . ) | (cd "${PATCH_DIR}/precommit"; tar xpf - ) + + if [[ -n "${USER_PLUGIN_DIR}" + && -d "${USER_PLUGIN_DIR}" ]]; then + cp -pr "${USER_PLUGIN_DIR}/*" \ + "${PATCH_DIR}/precommit/user-plugins" + fi + # Set to be relative to ${PATCH_DIR}/precommit + USER_PLUGIN_DIR="${PATCH_DIR}/precommit/user-plugins" + + if [[ -n ${PERSONALITY} + && -f ${PERSONALITY} ]]; then + cp -pr "${PERSONALITY}" "${PATCH_DIR}/precommit/personality" + person=$(basename "${PERSONALITY}") + + # Set to be relative to ${PATCH_DIR}/precommit + PERSONALITY="${PATCH_DIR}/precommit/personality/${person}" + fi + + if [[ -n ${DOCKERFILE} + && -f ${DOCKERFILE} ]]; then + dockerdir=$(dirname "${DOCKERFILE}") + dockfile=$(basename "${DOCKERFILE}") + pushd "${dockerdir}" >/dev/null + gitfilerev=$("${GIT}" log -n 1 --pretty=format:%h -- "${dockfile}" 2>/dev/null) + popd >/dev/null + if [[ -z ${gitfilerev} ]]; then + gitfilerev=$(date "+%F") + gitfilerev="date${gitfilerev}" + fi + ( + echo "### TEST_PATCH_PRIVATE: dockerfile=${DOCKERFILE}" + echo "### TEST_PATCH_PRIVATE: gitrev=${gitfilerev}" + cat "${DOCKERFILE}" + # make sure we put some space between, just in case last + # line isn't an empty line or whatever + printf "\n\n" + cat "${BINDIR}/test-patch-docker/Dockerfile-endstub" + + printf "\n\n" + ) > "${PATCH_DIR}/precommit/test-patch-docker/Dockerfile" + DOCKERFILE="${PATCH_DIR}/precommit/test-patch-docker/Dockerfile" + fi + + popd >/dev/null +} + +## @description If this patches actually patches test-patch.sh, then ## @description run with the patched version for the test. ## @audience private ## @stability evolving @@ -1483,241 +1724,664 @@ function apply_patch_file function check_reexec { local commentfile=${PATCH_DIR}/tp.${RANDOM} + local tpdir + local copy=false + local testdir + local person if [[ ${REEXECED} == true ]]; then big_console_header "Re-exec mode detected. Continuing." return fi - if [[ ! ${CHANGED_FILES} =~ dev-support/test-patch - && ! ${CHANGED_FILES} =~ dev-support/smart-apply ]] ; then + for testdir in "${BINDIR}" \ + "${PERSONALITY}" \ + "${USER_PLUGIN_DIR}" \ + "${DOCKERFILE}"; do + tpdir=$(relative_dir "${testdir}") + if [[ $? == 0 + && ${CHANGED_FILES} =~ ${tpdir} ]]; then + copy=true + fi + done + + if [[ ${copy} == true ]]; then + big_console_header "precommit patch detected" + + if [[ ${RESETREPO} == false ]]; then + ((RESULT = RESULT + 1)) + yetus_debug "can't destructively change the working directory. run with '--resetrepo' please. :(" + add_vote_table -1 precommit "Couldn't test precommit changes because we aren't configured to destructively change the working directory." + else + + apply_patch_file + + if [[ ${JENKINS} == true ]]; then + rm "${commentfile}" 2>/dev/null + echo "(!) A patch to the testing environment has been detected. " > "${commentfile}" + echo "Re-executing against the patched versions to perform further tests. " >> "${commentfile}" + echo "The console is at ${BUILD_URL}console in case of problems." >> "${commentfile}" + write_comment "${commentfile}" + rm "${commentfile}" + fi + fi + fi + + if [[ ${DOCKERSUPPORT} == false + && ${copy} == false ]]; then return fi - big_console_header "dev-support patch detected" + if [[ ${DOCKERSUPPORT} == true + && ${copy} == false ]]; then + big_console_header "Re-execing under Docker" - if [[ ${RESETREPO} == false ]]; then - ((RESULT = RESULT + 1)) - hadoop_debug "can't destructively change the working directory. run with '--resetrepo' please. :(" - add_jira_table -1 dev-support "Couldn't test dev-support changes because we aren't configured to destructively change the working directory." - return fi - printf "\n\nRe-executing against patched versions to test.\n\n" + # copy our universe + copytpbits - apply_patch_file + if [[ ${DOCKERSUPPORT} == true ]]; then + # if we are doing docker, then we re-exec, but underneath the + # container - if [[ ${JENKINS} == true ]]; then + client=$(docker version | grep 'Client version' | cut -f2 -d: | tr -d ' ') + server=$(docker version | grep 'Server version' | cut -f2 -d: | tr -d ' ') - rm "${commentfile}" 2>/dev/null + dockerversion="Client=${client} Server=${server}" - echo "(!) A patch to the files used for the QA process has been detected. " > "${commentfile}" - echo "Re-executing against the patched versions to perform further tests. " >> "${commentfile}" - echo "The console is at ${BUILD_URL}console in case of problems." >> "${commentfile}" + TESTPATCHMODE="${USER_PARAMS[*]}" + if [[ -n "${BUILD_URL}" ]]; then + TESTPATCHMODE="--build-url=${BUILD_URL} ${TESTPATCHMODE}" + fi + TESTPATCHMODE="--tpglobaltimer=${GLOBALTIMER} ${TESTPATCHMODE}" + TESTPATCHMODE="--tpreexectimer=${TIMER} ${TESTPATCHMODE}" + TESTPATCHMODE="--personality=\'${PERSONALITY}\' ${TESTPATCHMODE}" + TESTPATCHMODE="--plugins=\'${USER_PLUGIN_DIR}\' ${TESTPATCHMODE}" + TESTPATCHMODE=" ${TESTPATCHMODE}" + export TESTPATCHMODE + + patchdir=$(relative_dir "${PATCH_DIR}") + + cd "${BASEDIR}" + #shellcheck disable=SC2093 + exec bash "${PATCH_DIR}/precommit/test-patch-docker/test-patch-docker.sh" \ + --dockerversion="${dockerversion}" \ + --java-home="${JAVA_HOME}" \ + --patch-dir="${patchdir}" \ + --project="${PROJECT_NAME}" - write_to_jira "${commentfile}" - rm "${commentfile}" + else + + # if we aren't doing docker, then just call ourselves + # but from the new path with the new flags + #shellcheck disable=SC2093 + cd "${PATCH_DIR}/precommit/" + exec "${PATCH_DIR}/precommit/test-patch.sh" \ + "${USER_PARAMS[@]}" \ + --reexec \ + --basedir="${BASEDIR}" \ + --branch="${PATCH_BRANCH}" \ + --patch-dir="${PATCH_DIR}" \ + --tpglobaltimer="${GLOBALTIMER}" \ + --tpreexectimer="${TIMER}" \ + --personality="${PERSONALITY}" \ + --plugins="${USER_PLUGIN_DIR}" + fi +} + +## @description Reset the test results +## @audience public +## @stability evolving +## @replaceable no +function modules_reset +{ + MODULE_STATUS=() + MODULE_STATUS_TIMER=() + MODULE_STATUS_MSG=() + MODULE_STATUS_LOG=() +} + +## @description Utility to print standard module errors +## @audience public +## @stability evolving +## @replaceable no +## @param repostatus +## @param testtype +## @param mvncmdline +function modules_messages +{ + local repostatus=$1 + local testtype=$2 + local summarymode=$3 + shift 2 + local modindex=0 + local repo + local goodtime=0 + local failure=false + local oldtimer + local statusjdk + local multijdkmode=false + + if [[ ${repostatus} == branch ]]; then + repo=${PATCH_BRANCH} + else + repo="the patch" + fi + + verify_multijdk_test "${testtype}" + if [[ $? == 1 ]]; then + multijdkmode=true + fi + + oldtimer=${TIMER} + + if [[ ${summarymode} == true + && ${ALLOWSUMMARIES} == true ]]; then + + until [[ ${modindex} -eq ${#MODULE[@]} ]]; do + + if [[ ${multijdkmode} == true ]]; then + statusjdk=${MODULE_STATUS_JDK[${modindex}]} + fi + + if [[ "${MODULE_STATUS[${modindex}]}" == '+1' ]]; then + ((goodtime=goodtime + ${MODULE_STATUS_TIMER[${modindex}]})) + else + failure=true + start_clock + echo "" + echo "${MODULE_STATUS_MSG[${modindex}]}" + echo "" + offset_clock "${MODULE_STATUS_TIMER[${modindex}]}" + add_vote_table "${MODULE_STATUS[${modindex}]}" "${testtype}" "${MODULE_STATUS_MSG[${modindex}]}" + if [[ ${MODULE_STATUS[${modindex}]} == -1 + && -n "${MODULE_STATUS_LOG[${modindex}]}" ]]; then + add_footer_table "${testtype}" "@@BASE@@/${MODULE_STATUS_LOG[${modindex}]}" + fi + fi + ((modindex=modindex+1)) + done + + if [[ ${failure} == false ]]; then + start_clock + offset_clock "${goodtime}" + add_vote_table +1 "${testtype}" "${repo} passed${statusjdk}" + fi + else + until [[ ${modindex} -eq ${#MODULE[@]} ]]; do + start_clock + echo "" + echo "${MODULE_STATUS_MSG[${modindex}]}" + echo "" + offset_clock "${MODULE_STATUS_TIMER[${modindex}]}" + add_vote_table "${MODULE_STATUS[${modindex}]}" "${testtype}" "${MODULE_STATUS_MSG[${modindex}]}" + if [[ ${MODULE_STATUS[${modindex}]} == -1 + && -n "${MODULE_STATUS_LOG[${modindex}]}" ]]; then + add_footer_table "${testtype}" "@@BASE@@/${MODULE_STATUS_LOG[${modindex}]}" + fi + ((modindex=modindex+1)) + done fi + TIMER=${oldtimer} +} + +## @description Add a test result +## @audience public +## @stability evolving +## @replaceable no +## @param module +## @param runtime +function module_status +{ + local index=$1 + local value=$2 + local log=$3 + shift 3 - cd "${CWD}" - mkdir -p "${PATCH_DIR}/dev-support-test" - cp -pr "${BASEDIR}"/dev-support/test-patch* "${PATCH_DIR}/dev-support-test" - cp -pr "${BASEDIR}"/dev-support/smart-apply* "${PATCH_DIR}/dev-support-test" + local jdk - big_console_header "exec'ing test-patch.sh now..." + jdk=$(report_jvm_version "${JAVA_HOME}") - exec "${PATCH_DIR}/dev-support-test/test-patch.sh" \ - --reexec \ - --branch="${PATCH_BRANCH}" \ - --patch-dir="${PATCH_DIR}" \ - "${USER_PARAMS[@]}" + if [[ -n ${index} + && ${index} =~ ^[0-9]+$ ]]; then + MODULE_STATUS[${index}]="${value}" + MODULE_STATUS_LOG[${index}]="${log}" + MODULE_STATUS_JDK[${index}]=" with JDK v${jdk}" + MODULE_STATUS_MSG[${index}]="${*}" + else + yetus_error "ASSERT: module_status given bad index: ${index}" + local frame=0 + while caller $frame; do + ((frame++)); + done + echo "$*" + exit 1 + fi } -## @description Check the current directory for @author tags -## @audience private +## @description run the maven tests for the queued modules +## @audience public +## @stability evolving +## @replaceable no +## @param repostatus +## @param testtype +## @param mvncmdline +function modules_workers +{ + local repostatus=$1 + local testtype=$2 + shift 2 + local modindex=0 + local fn + local savestart=${TIMER} + local savestop + local repo + local modulesuffix + local jdk="" + local jdkindex=0 + local statusjdk + + if [[ ${repostatus} == branch ]]; then + repo=${PATCH_BRANCH} + else + repo="the patch" + fi + + modules_reset + + verify_multijdk_test "${testtype}" + if [[ $? == 1 ]]; then + jdk=$(report_jvm_version "${JAVA_HOME}") + statusjdk=" with JDK v${jdk}" + jdk="-jdk${jdk}" + jdk=${jdk// /} + yetus_debug "Starting MultiJDK mode${statusjdk} on ${testtype}" + fi + + until [[ ${modindex} -eq ${#MODULE[@]} ]]; do + start_clock + + fn=$(module_file_fragment "${MODULE[${modindex}]}") + fn="${fn}${jdk}" + modulesuffix=$(basename "${MODULE[${modindex}]}") + pushd "${BASEDIR}/${MODULE[${modindex}]}" >/dev/null + + if [[ ${modulesuffix} == . ]]; then + modulesuffix="root" + fi + + if [[ $? != 0 ]]; then + echo "${BASEDIR}/${MODULE[${modindex}]} no longer exists. Skipping." + ((modindex=modindex+1)) + continue + fi + + case ${BUILDTOOL} in + maven) + #shellcheck disable=SC2086 + echo_and_redirect "${PATCH_DIR}/${repostatus}-${testtype}-${fn}.txt" \ + ${MVN} "${MAVEN_ARGS[@]}" \ + "${@//@@@MODULEFN@@@/${fn}}" \ + ${MODULEEXTRAPARAM[${modindex}]//@@@MODULEFN@@@/${fn}} -Ptest-patch + ;; + ant) + #shellcheck disable=SC2086 + echo_and_redirect "${PATCH_DIR}/${repostatus}-${testtype}-${fn}.txt" \ + "${ANT}" "${ANT_ARGS[@]}" \ + ${MODULEEXTRAPARAM[${modindex}]//@@@MODULEFN@@@/${fn}} \ + "${@//@@@MODULEFN@@@/${fn}}" + ;; + *) + yetus_error "ERROR: Unsupported build tool." + return 1 + ;; + esac + + if [[ $? == 0 ]] ; then + module_status \ + ${modindex} \ + +1 \ + "${repostatus}-${testtype}-${fn}.txt" \ + "${modulesuffix} in ${repo} passed${statusjdk}." + else + module_status \ + ${modindex} \ + -1 \ + "${repostatus}-${testtype}-${fn}.txt" \ + "${modulesuffix} in ${repo} failed${statusjdk}." + ((result = result + 1)) + fi + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${modindex}]=${savestop} + # shellcheck disable=SC2086 + echo "Elapsed: $(clock_display ${savestop})" + popd >/dev/null + ((modindex=modindex+1)) + done + + TIMER=${savestart} + + if [[ ${result} -gt 0 ]]; then + return 1 + fi + return 0 +} + +## @description Reset the queue for tests +## @audience public +## @stability evolving +## @replaceable no +function clear_personality_queue +{ + yetus_debug "Personality: clear queue" + MODCOUNT=0 + MODULE=() +} + +## @description Build the queue for tests +## @audience public ## @stability evolving ## @replaceable no +## @param module +## @param profiles/flags/etc +function personality_enqueue_module +{ + yetus_debug "Personality: enqueue $*" + local module=$1 + shift + + MODULE[${MODCOUNT}]=${module} + MODULEEXTRAPARAM[${MODCOUNT}]=${*} + ((MODCOUNT=MODCOUNT+1)) +} + +## @description Confirm compilation pre-patch +## @audience private +## @stability stable +## @replaceable no ## @return 0 on success ## @return 1 on failure -function check_author +function precheck_javac { - local authorTags + local result=0 + local -r savejavahome=${JAVA_HOME} + local multijdkmode=false + local jdkindex=0 - big_console_header "Checking there are no @author tags in the patch." + big_console_header "Pre-patch ${PATCH_BRANCH} javac compilation" - start_clock + verify_needed_test javac + if [[ $? == 0 ]]; then + echo "Patch does not appear to need javac tests." + return 0 + fi - if [[ ${CHANGED_FILES} =~ dev-support/test-patch ]]; then - add_jira_table 0 @author "Skipping @author checks as test-patch has been patched." - return 0 + verify_multijdk_test javac + if [[ $? == 1 ]]; then + multijdkmode=true fi - authorTags=$("${GREP}" -c -i '^[^-].*@author' "${PATCH_DIR}/patch") - echo "There appear to be ${authorTags} @author tags in the patch." - if [[ ${authorTags} != 0 ]] ; then - add_jira_table -1 @author \ - "The patch appears to contain ${authorTags} @author tags which the Hadoop" \ - " community has agreed to not allow in code contributions." + for jdkindex in ${JDK_DIR_LIST}; do + if [[ ${multijdkmode} == true ]]; then + JAVA_HOME=${jdkindex} + fi + + personality_modules branch javac + case ${BUILDTOOL} in + maven) + modules_workers branch javac clean compile + ;; + ant) + modules_workers branch javac + ;; + *) + yetus_error "ERROR: Unsupported build tool." + return 1 + ;; + esac + + ((result=result + $?)) + modules_messages branch javac true + + done + JAVA_HOME=${savejavahome} + + if [[ ${result} -gt 0 ]]; then return 1 fi - add_jira_table +1 @author "The patch does not contain any @author tags." return 0 } -## @description Check the patch file for changed/new tests +## @description Confirm Javadoc pre-patch ## @audience private -## @stability evolving +## @stability stable ## @replaceable no ## @return 0 on success ## @return 1 on failure -function check_modified_unittests +function precheck_javadoc { - local testReferences=0 - local i + local result=0 + local -r savejavahome=${JAVA_HOME} + local multijdkmode=false + local jdkindex=0 - verify_needed_test unit + big_console_header "Pre-patch ${PATCH_BRANCH} Javadoc verification" + verify_needed_test javadoc if [[ $? == 0 ]]; then - return 0 + echo "Patch does not appear to need javadoc tests." + return 0 fi - big_console_header "Checking there are new or changed tests in the patch." - - start_clock + verify_multijdk_test javadoc + if [[ $? == 1 ]]; then + multijdkmode=true + fi - for i in ${CHANGED_FILES}; do - if [[ ${i} =~ /test/ ]]; then - ((testReferences=testReferences + 1)) + for jdkindex in ${JDK_DIR_LIST}; do + if [[ ${multijdkmode} == true ]]; then + JAVA_HOME=${jdkindex} fi + + personality_modules branch javadoc + case ${BUILDTOOL} in + maven) + modules_workers branch javadoc clean javadoc:javadoc + ;; + ant) + modules_workers branch javadoc clean javadoc + ;; + *) + yetus_error "ERROR: Unsupported build tool." + return 1 + ;; + esac + + ((result=result + $?)) + modules_messages branch javadoc true + done + JAVA_HOME=${savejavahome} - echo "There appear to be ${testReferences} test file(s) referenced in the patch." - if [[ ${testReferences} == 0 ]] ; then - add_jira_table -1 "tests included" \ - "The patch doesn't appear to include any new or modified tests. " \ - "Please justify why no new tests are needed for this patch." \ - "Also please list what manual steps were performed to verify this patch." + if [[ ${result} -gt 0 ]]; then return 1 fi - add_jira_table +1 "tests included" \ - "The patch appears to include ${testReferences} new or modified test files." return 0 } -## @description Helper for check_javadoc +## @description Confirm site pre-patch ## @audience private -## @stability evolving +## @stability stable ## @replaceable no ## @return 0 on success ## @return 1 on failure -function count_javadoc_warns +function precheck_site { - local warningfile=$1 + local result=0 - #shellcheck disable=SC2016,SC2046 - return $(${EGREP} "^[0-9]+ warnings$" "${warningfile}" | ${AWK} '{sum+=$1} END {print sum}') + if [[ ${BUILDTOOL} != maven ]]; then + return 0 + fi + + big_console_header "Pre-patch ${PATCH_BRANCH} site verification" + + verify_needed_test site + if [[ $? == 0 ]];then + echo "Patch does not appear to need site tests." + return 0 + fi + + personality_modules branch site + modules_workers branch site clean site site:stage + result=$? + modules_messages branch site true + if [[ ${result} != 0 ]]; then + return 1 + fi + return 0 } -## @description Count and compare the number of JavaDoc warnings pre- and post- patch +## @description Confirm the source environment pre-patch +## @audience private +## @stability stable +## @replaceable no +## @return 0 on success +## @return 1 on failure +function precheck_without_patch +{ + local result=0 + + precheck_mvninstall + + if [[ $? -gt 0 ]]; then + ((result = result +1 )) + fi + + precheck_javac + + if [[ $? -gt 0 ]]; then + ((result = result +1 )) + fi + + precheck_javadoc + + if [[ $? -gt 0 ]]; then + ((result = result +1 )) + fi + + precheck_site + + if [[ $? -gt 0 ]]; then + ((result = result +1 )) + fi + + if [[ ${result} -gt 0 ]]; then + return 1 + fi + + return 0 +} + +## @description Check the current directory for @author tags ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success ## @return 1 on failure -function check_javadoc +function check_author { - local numBranchJavadocWarnings - local numPatchJavadocWarnings + local authorTags + local -r appname=$(basename "${BASH_SOURCE-$0}") - verify_needed_test javadoc + big_console_header "Checking there are no @author tags in the patch." - if [[ $? == 0 ]]; then - echo "This patch does not appear to need javadoc checks." + if [[ ${CHANGED_FILES} =~ ${appname} ]]; then + echo "Skipping @author checks as ${appname} has been patched." + add_vote_table 0 @author "Skipping @author checks as ${appname} has been patched." return 0 fi - big_console_header "Determining number of patched javadoc warnings" - start_clock - if [[ -d hadoop-project ]]; then - (cd hadoop-project; "${MVN}" "${MAVEN_ARGS[@]}" install > /dev/null 2>&1) - fi - if [[ -d hadoop-common-project/hadoop-annotations ]]; then - (cd hadoop-common-project/hadoop-annotations; "${MVN}" "${MAVEN_ARGS[@]}" install > /dev/null 2>&1) - fi - echo_and_redirect "${PATCH_DIR}/patchJavadocWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess - count_javadoc_warns "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt" - numBranchJavadocWarnings=$? - count_javadoc_warns "${PATCH_DIR}/patchJavadocWarnings.txt" - numPatchJavadocWarnings=$? - - echo "There appear to be ${numBranchJavadocWarnings} javadoc warnings before the patch and ${numPatchJavadocWarnings} javadoc warnings after applying the patch." - if [[ ${numBranchJavadocWarnings} != "" && ${numPatchJavadocWarnings} != "" ]] ; then - if [[ ${numPatchJavadocWarnings} -gt ${numBranchJavadocWarnings} ]] ; then - - ${GREP} -i warning "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt" > "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt" - ${GREP} -i warning "${PATCH_DIR}/patchJavadocWarnings.txt" > "${PATCH_DIR}/patchJavadocWarningsFiltered.txt" - ${DIFF} -u "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt" \ - "${PATCH_DIR}/patchJavadocWarningsFiltered.txt" \ - > "${PATCH_DIR}/diffJavadocWarnings.txt" - rm -f "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt" "${PATCH_DIR}/patchJavadocWarningsFiltered.txt" - - add_jira_table -1 javadoc "The applied patch generated "\ - "$((numPatchJavadocWarnings-numBranchJavadocWarnings))" \ - " additional warning messages." - add_jira_footer javadoc "@@BASE@@/diffJavadocWarnings.txt" - return 1 - fi + authorTags=$("${GREP}" -c -i '^[^-].*@author' "${PATCH_DIR}/patch") + echo "There appear to be ${authorTags} @author tags in the patch." + if [[ ${authorTags} != 0 ]] ; then + add_vote_table -1 @author \ + "The patch appears to contain ${authorTags} @author tags which the" \ + " community has agreed to not allow in code contributions." + return 1 fi - add_jira_table +1 javadoc "There were no new javadoc warning messages." + add_vote_table +1 @author "The patch does not contain any @author tags." return 0 } -## @description Make sure site still compiles +## @description Check the patch file for changed/new tests ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success ## @return 1 on failure -function check_site +function check_modified_unittests { - local -r mypwd=$(pwd) + local testReferences=0 + local i - verify_needed_test site + big_console_header "Checking there are new or changed tests in the patch." + + verify_needed_test unit if [[ $? == 0 ]]; then - echo "This patch does not appear to need site checks." + echo "Patch does not appear to need new or modified tests." return 0 fi - big_console_header "Determining if patched site still builds" - start_clock - echo "site creation for ${mypwd}" - echo_and_redirect "${PATCH_DIR}/patchSiteWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess - if [[ $? != 0 ]] ; then - echo "Site compilation is broken" - add_jira_table -1 site "Site compilation is broken." - add_jira_footer site "@@BASE@@/patchSiteWarnings.txt" + for i in ${CHANGED_FILES}; do + if [[ ${i} =~ /test/ ]]; then + ((testReferences=testReferences + 1)) + fi + done + + echo "There appear to be ${testReferences} test file(s) referenced in the patch." + if [[ ${testReferences} == 0 ]] ; then + add_vote_table -1 "test4tests" \ + "The patch doesn't appear to include any new or modified tests. " \ + "Please justify why no new tests are needed for this patch." \ + "Also please list what manual steps were performed to verify this patch." return 1 fi - add_jira_table +1 site "Site still builds." + add_vote_table +1 "test4tests" \ + "The patch appears to include ${testReferences} new or modified test files." return 0 } -## @description Helper for check_javac +## @description Helper for check_patch_javac ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success ## @return 1 on failure -function count_javac_warns +function count_javac_probs { local warningfile=$1 - #shellcheck disable=SC2016,SC2046 - return $(${AWK} 'BEGIN {total = 0} {total += 1} END {print total}' "${warningfile}") + local val1 + local val2 + + case ${BUILDTOOL} in + maven) + #shellcheck disable=SC2016,SC2046 + ${AWK} 'BEGIN {total = 0} {total += 1} END {print total}' "${warningfile}" + ;; + ant) + #shellcheck disable=SC2016 + val1=$(${GREP} -E "\[javac\] [0-9]+ errors?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}') + #shellcheck disable=SC2016 + val2=$(${GREP} -E "\[javac\] [0-9]+ warnings?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}') + echo $((val1+val2)) + ;; + esac } ## @description Count and compare the number of javac warnings pre- and post- patch @@ -1726,387 +2390,352 @@ function count_javac_warns ## @replaceable no ## @return 0 on success ## @return 1 on failure -function check_javac +function check_patch_javac { - local branchJavacWarnings - local patchJavacWarnings + local i + local result=0 + local fn + local -r savejavahome=${JAVA_HOME} + local multijdkmode=false + local jdk="" + local jdkindex=0 + local statusjdk + declare -i numbranch=0 + declare -i numpatch=0 + + big_console_header "Determining number of patched javac errors" verify_needed_test javac if [[ $? == 0 ]]; then - echo "This patch does not appear to need javac checks." + echo "Patch does not appear to need javac tests." return 0 fi - big_console_header "Determining number of patched javac warnings." + verify_multijdk_test javac + if [[ $? == 1 ]]; then + multijdkmode=true + fi - start_clock + for jdkindex in ${JDK_DIR_LIST}; do + if [[ ${multijdkmode} == true ]]; then + JAVA_HOME=${jdkindex} + jdk=$(report_jvm_version "${JAVA_HOME}") + yetus_debug "Using ${JAVA_HOME} to run this set of tests" + statusjdk=" with JDK v${jdk}" + jdk="-jdk${jdk}" + jdk=${jdk// /} + fi - echo_and_redirect "${PATCH_DIR}/patchJavacWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess ${NATIVE_PROFILE} -Ptest-patch - if [[ $? != 0 ]] ; then - add_jira_table -1 javac "The patch appears to cause the build to fail." - return 2 - fi - ### Compare ${PATCH_BRANCH} and patch javac warning numbers - if [[ -f ${PATCH_DIR}/patchJavacWarnings.txt ]] ; then - ${GREP} '\[WARNING\]' "${PATCH_DIR}/${PATCH_BRANCH}JavacWarnings.txt" > "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt" - ${GREP} '\[WARNING\]' "${PATCH_DIR}/patchJavacWarnings.txt" > "${PATCH_DIR}/filteredPatchJavacWarnings.txt" + personality_modules patch javac + + case ${BUILDTOOL} in + maven) + modules_workers patch javac clean compile + ;; + ant) + modules_workers patch javac + ;; + *) + yetus_error "ERROR: Unsupported build tool." + return 1 + ;; + esac - count_javac_warns "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt" - branchJavacWarnings=$? - count_javac_warns "${PATCH_DIR}/filteredPatchJavacWarnings.txt" - patchJavacWarnings=$? + i=0 + until [[ ${i} -eq ${#MODULE[@]} ]]; do + if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then + ((result=result+1)) + ((i=i+1)) + continue + fi - echo "There appear to be ${branchJavacWarnings} javac compiler warnings before the patch and ${patchJavacWarnings} javac compiler warnings after applying the patch." - if [[ ${patchJavacWarnings} != "" && ${branchJavacWarnings} != "" ]] ; then - if [[ ${patchJavacWarnings} -gt ${branchJavacWarnings} ]] ; then + fn=$(module_file_fragment "${MODULE[${i}]}") + fn="${fn}${jdk}" + module_suffix=$(basename "${MODULE[${i}]}") + if [[ ${module_suffix} == \. ]]; then + module_suffix=root + fi - ${DIFF} "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt" \ - "${PATCH_DIR}/filteredPatchJavacWarnings.txt" \ - > "${PATCH_DIR}/diffJavacWarnings.txt" + # if it was a new module, this won't exist. + if [[ -f "${PATCH_DIR}/branch-javac-${fn}.txt" ]]; then + ${GREP} '\[WARNING\]' "${PATCH_DIR}/branch-javac-${fn}.txt" \ + > "${PATCH_DIR}/branch-javac-${fn}-warning.txt" + else + touch "${PATCH_DIR}/branch-javac-${fn}.txt" \ + "${PATCH_DIR}/branch-javac-${fn}-warning.txt" + fi - add_jira_table -1 javac "The applied patch generated "\ - "$((patchJavacWarnings-branchJavacWarnings))" \ - " additional warning messages." + ${GREP} '\[WARNING\]' "${PATCH_DIR}/patch-javac-${fn}.txt" \ + > "${PATCH_DIR}/patch-javac-${fn}-warning.txt" - add_jira_footer javac "@@BASE@@/diffJavacWarnings.txt" + numbranch=$(count_javac_probs "${PATCH_DIR}/branch-javac-${fn}-warning.txt") + numpatch=$(count_javac_probs "${PATCH_DIR}/patch-javac-${fn}-warning.txt") - return 1 + if [[ -n ${numbranch} + && -n ${numpatch} + && ${numpatch} -gt ${numbranch} ]]; then + + ${DIFF} -u "${PATCH_DIR}/branch-javac-${fn}-warning.txt" \ + "${PATCH_DIR}/patch-javac-${fn}-warning.txt" \ + > "${PATCH_DIR}/javac-${fn}-diff.txt" + + module_status ${i} -1 "javac-${fn}-diff.txt" \ + "Patched ${module_suffix} generated "\ + "$((numpatch-numbranch)) additional warning messages${statusjdk}." \ + + ((result=result+1)) fi - fi - fi + ((i=i+1)) + done - add_jira_table +1 javac "There were no new javac warning messages." + modules_messages patch javac true + done + JAVA_HOME=${savejavahome} + + if [[ ${result} -gt 0 ]]; then + return 1 + fi return 0 } -## @description Verify all files have an Apache License +## @description Helper for check_patch_javadoc ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success ## @return 1 on failure -function check_apachelicense +function count_javadoc_probs { - big_console_header "Determining number of patched release audit warnings." - - start_clock - - echo_and_redirect "${PATCH_DIR}/patchReleaseAuditOutput.txt" "${MVN}" "${MAVEN_ARGS[@]}" apache-rat:check -D${PROJECT_NAME}PatchProcess - #shellcheck disable=SC2038 - find "${BASEDIR}" -name rat.txt | xargs cat > "${PATCH_DIR}/patchReleaseAuditWarnings.txt" - - ### Compare ${PATCH_BRANCH} and patch release audit warning numbers - if [[ -f ${PATCH_DIR}/patchReleaseAuditWarnings.txt ]] ; then - patchReleaseAuditWarnings=$("${GREP}" -c '\!?????' "${PATCH_DIR}/patchReleaseAuditWarnings.txt") - echo "" - echo "" - echo "There appear to be ${patchReleaseAuditWarnings} release audit warnings after applying the patch." - if [[ ${patchReleaseAuditWarnings} != "" ]] ; then - if [[ ${patchReleaseAuditWarnings} -gt 0 ]] ; then - add_jira_table -1 "release audit" "The applied patch generated ${patchReleaseAuditWarnings} release audit warnings." - - ${GREP} '\!?????' "${PATCH_DIR}/patchReleaseAuditWarnings.txt" \ - > "${PATCH_DIR}/patchReleaseAuditProblems.txt" - - echo "Lines that start with ????? in the release audit "\ - "report indicate files that do not have an Apache license header." \ - >> "${PATCH_DIR}/patchReleaseAuditProblems.txt" - - add_jira_footer "Release Audit" "@@BASE@@/patchReleaseAuditProblems.txt" + local warningfile=$1 + local val1 + local val2 - return 1 - fi - fi - fi - add_jira_table 1 "release audit" "The applied patch does not increase the total number of release audit warnings." - return 0 + case ${BUILDTOOL} in + maven) + #shellcheck disable=SC2016,SC2046 + ${GREP} -E "^[0-9]+ warnings?$" "${warningfile}" | ${AWK} '{sum+=$1} END {print sum}' + ;; + ant) + #shellcheck disable=SC2016 + val1=$(${GREP} -E "\[javadoc\] [0-9]+ errors?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}') + #shellcheck disable=SC2016 + val2=$(${GREP} -E "\[javadoc\] [0-9]+ warnings?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}') + echo $((val1+val2)) + ;; + esac } -## @description Verify mvn install works +## @description Count and compare the number of JavaDoc warnings pre- and post- patch ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success ## @return 1 on failure -function check_mvn_install +function check_patch_javadoc { - local retval + local i + local result=0 + local fn + local -r savejavahome=${JAVA_HOME} + local multijdkmode=false + local jdk="" + local jdkindex=0 + local statusjdk + declare -i numbranch=0 + declare -i numpatch=0 + + big_console_header "Determining number of patched javadoc warnings" verify_needed_test javadoc - retval=$? + if [[ $? == 0 ]]; then + echo "Patch does not appear to need javadoc tests." + return 0 + fi - verify_needed_test javac - ((retval = retval + $? )) + verify_multijdk_test javadoc + if [[ $? == 1 ]]; then + multijdkmode=true + fi + + for jdkindex in ${JDK_DIR_LIST}; do + if [[ ${multijdkmode} == true ]]; then + JAVA_HOME=${jdkindex} + jdk=$(report_jvm_version "${JAVA_HOME}") + yetus_debug "Using ${JAVA_HOME} to run this set of tests" + statusjdk=" with JDK v${jdk}" + jdk="-jdk${jdk}" + jdk=${jdk// /} + fi + + personality_modules patch javadoc + case ${BUILDTOOL} in + maven) + modules_workers patch javadoc clean javadoc:javadoc + ;; + ant) + modules_workers patch javadoc clean javadoc + ;; + *) + yetus_error "ERROR: Unsupported build tool." + return 1 + ;; + esac + + i=0 + until [[ ${i} -eq ${#MODULE[@]} ]]; do + if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then + ((result=result+1)) + ((i=i+1)) + continue + fi + + fn=$(module_file_fragment "${MODULE[${i}]}") + fn="${fn}${jdk}" + numbranch=$(count_javadoc_probs "${PATCH_DIR}/branch-javadoc-${fn}.txt") + numpatch=$(count_javadoc_probs "${PATCH_DIR}/patch-javadoc-${fn}.txt") + + if [[ -n ${numbranch} + && -n ${numpatch} + && ${numpatch} -gt ${numbranch} ]] ; then + + if [[ -f "${PATCH_DIR}/branch-javadoc-${fn}.txt" ]]; then + ${GREP} -i warning "${PATCH_DIR}/branch-javadoc-${fn}.txt" \ + > "${PATCH_DIR}/branch-javadoc-${fn}-filtered.txt" + else + touch "${PATCH_DIR}/branch-javadoc-${fn}.txt" \ + "${PATCH_DIR}/b
<TRUNCATED>
