Author: nigel Date: Tue Nov 2 05:33:05 2010 New Revision: 1029958 URL: http://svn.apache.org/viewvc?rev=1029958&view=rev Log: HADOOP-7008. Enable test-patch.sh to have a configured number of acceptable findbugs and javadoc warnings. Contributed by nigel and gkesavan.
Added: hadoop/common/trunk/src/test/bin/test-patch.properties Modified: hadoop/common/trunk/CHANGES.txt hadoop/common/trunk/src/test/bin/test-patch.sh Modified: hadoop/common/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=1029958&r1=1029957&r2=1029958&view=diff ============================================================================== --- hadoop/common/trunk/CHANGES.txt (original) +++ hadoop/common/trunk/CHANGES.txt Tue Nov 2 05:33:05 2010 @@ -155,6 +155,9 @@ Trunk (unreleased changes) HADOOP-7009. MD5Hash provides a public factory method that creates an instance of thread local MessageDigest. (hairong) + HADOOP-7008. Enable test-patch.sh to have a configured number of acceptable + findbugs and javadoc warnings. (nigel and gkesavan) + OPTIMIZATIONS HADOOP-6884. Add LOG.isDebugEnabled() guard for each LOG.debug(..). Added: hadoop/common/trunk/src/test/bin/test-patch.properties URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/bin/test-patch.properties?rev=1029958&view=auto ============================================================================== --- hadoop/common/trunk/src/test/bin/test-patch.properties (added) +++ hadoop/common/trunk/src/test/bin/test-patch.properties Tue Nov 2 05:33:05 2010 @@ -0,0 +1,3 @@ +OK_RELEASEAUDIT_WARNINGS=1 +OK_FINDBUGS_WARNINGS=0 +OK_JAVADOC_WARNINGS=6 Modified: hadoop/common/trunk/src/test/bin/test-patch.sh URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/bin/test-patch.sh?rev=1029958&r1=1029957&r2=1029958&view=diff ============================================================================== --- hadoop/common/trunk/src/test/bin/test-patch.sh (original) +++ hadoop/common/trunk/src/test/bin/test-patch.sh Tue Nov 2 05:33:05 2010 @@ -17,6 +17,8 @@ ulimit -n 1024 ### Setup some variables. ### SVN_REVISION and BUILD_URL are set by Hudson if it is run by patch process +### Read variables from properties file +. `dirname $0`/test-patch.properties ############################################################################### parseArgs() { @@ -162,31 +164,29 @@ setup () { cleanupAndExit 0 fi fi + ### exit if warnings are NOT defined in the properties file + if [ -z "$OK_FINDBUGS_WARNINGS" ] || [[ -z "$OK_JAVADOC_WARNINGS" ]] || [[ -z $OK_RELEASEAUDIT_WARNINGS ]]; then + echo "Please define the following properties in test-patch.properties file" + echo "OK_FINDBUGS_WARNINGS" + echo "OK_RELEASEAUDIT_WARNINGS" + echo "OK_JAVAFINDBUGS_WARNINGS" + cleanupAndExit 1 + fi echo "" echo "" echo "======================================================================" echo "======================================================================" - echo " Pre-building trunk to determine trunk number" - echo " of release audit, javac, and Findbugs warnings." + echo " Pre-build trunk to verify trunk stability and javac warnings" echo "======================================================================" echo "======================================================================" echo "" echo "" - echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -Djava5.home=${JAVA5_HOME} -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= releaseaudit > $PATCH_DIR/trunkReleaseAuditWarnings.txt 2>&1" - $ANT_HOME/bin/ant -Dversion="${VERSION}" -Djava5.home=${JAVA5_HOME} -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= releaseaudit > $PATCH_DIR/trunkReleaseAuditWarnings.txt 2>&1 - echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -Djavac.args="-Xlint -Xmaxwarns 1000" $ECLIPSE_PROPERTY -Djava5.home=${JAVA5_HOME} -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= clean tar > $PATCH_DIR/trunkJavacWarnings.txt 2>&1" - $ANT_HOME/bin/ant -Dversion="${VERSION}" -Djavac.args="-Xlint -Xmaxwarns 1000" $ECLIPSE_PROPERTY -Djava5.home=${JAVA5_HOME} -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= clean tar > $PATCH_DIR/trunkJavacWarnings.txt 2>&1 + echo "$ANT_HOME/bin/ant -Djavac.args="-Xlint -Xmaxwarns 1000" $ECLIPSE_PROPERTY -Djava5.home=${JAVA5_HOME} -Dforrest.home=${FORREST_HOME} -D${PROJECT_NAME}PatchProcess= clean tar > $PATCH_DIR/trunkJavacWarnings.txt 2>&1" + $ANT_HOME/bin/ant -Djavac.args="-Xlint -Xmaxwarns 1000" $ECLIPSE_PROPERTY -Djava5.home=${JAVA5_HOME} -Dforrest.home=${FORREST_HOME} -D${PROJECT_NAME}PatchProcess= clean tar > $PATCH_DIR/trunkJavacWarnings.txt 2>&1 if [[ $? != 0 ]] ; then echo "Trunk compilation is broken?" cleanupAndExit 1 fi - echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -Dfindbugs.home=$FINDBUGS_HOME -Djava5.home=${JAVA5_HOME} -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= findbugs > /dev/null 2>&1" - $ANT_HOME/bin/ant -Dversion="${VERSION}" -Dfindbugs.home=$FINDBUGS_HOME -Djava5.home=${JAVA5_HOME} -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= findbugs > /dev/null 2>&1 - if [[ $? != 0 ]] ; then - echo "Trunk findbugs is broken?" - cleanupAndExit 1 - fi - cp $BASEDIR/build/test/findbugs/*.xml $PATCH_DIR/trunkFindbugsWarnings.xml } ############################################################################### @@ -299,20 +299,22 @@ checkJavadocWarnings () { echo "" echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= clean javadoc | tee $PATCH_DIR/patchJavadocWarnings.txt" $ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= clean javadoc | tee $PATCH_DIR/patchJavadocWarnings.txt - javadocWarnings=`$GREP -c '\[javadoc\] [0-9]* warning' $PATCH_DIR/patchJavadocWarnings.txt` + javadocWarnings=`$GREP -o '\[javadoc\] [0-9]* warning' $PATCH_DIR/patchJavadocWarnings.txt | awk '{total += $2} END {print total}'` echo "" echo "" echo "There appear to be $javadocWarnings javadoc warnings generated by the patched build." - if [[ $javadocWarnings != 0 ]] ; then + + ### if current warnings greater than OK_JAVADOC_WARNINGS + if [[ $javadocWarnings > $OK_JAVADOC_WARNINGS ]] ; then JIRA_COMMENT="$JIRA_COMMENT - -1 javadoc. The javadoc tool appears to have generated $javadocWarnings warning messages." + -1 javadoc. The javadoc tool appears to have generated `expr $(($javadocWarnings-$OK_JAVADOC_WARNINGS))` warning messages." return 1 fi JIRA_COMMENT="$JIRA_COMMENT +1 javadoc. The javadoc tool did not generate any warning messages." -return 0 + return 0 } ############################################################################### @@ -372,23 +374,18 @@ checkReleaseAuditWarnings () { ### Compare trunk and patch release audit warning numbers if [[ -f $PATCH_DIR/patchReleaseAuditWarnings.txt ]] ; then - trunkReleaseAuditWarnings=`$GREP -c '\!?????' $PATCH_DIR/trunkReleaseAuditWarnings.txt` patchReleaseAuditWarnings=`$GREP -c '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt` echo "" echo "" - echo "There appear to be $trunkReleaseAuditWarnings release audit warnings before the patch and $patchReleaseAuditWarnings release audit warnings after applying the patch." - if [[ $patchReleaseAuditWarnings != "" && $trunkReleaseAuditWarnings != "" ]] ; then - if [[ $patchReleaseAuditWarnings -gt $trunkReleaseAuditWarnings ]] ; then + echo "There appear to be $OK_RELEASEAUDIT_WARNINGS release audit warnings before the patch and $patchReleaseAuditWarnings release audit warnings after applying the patch." + if [[ $patchReleaseAuditWarnings != "" && $OK_RELEASEAUDIT_WARNINGS != "" ]] ; then + if [[ $patchReleaseAuditWarnings -gt $OK_RELEASEAUDIT_WARNINGS ]] ; then JIRA_COMMENT="$JIRA_COMMENT - -1 release audit. The applied patch generated $patchReleaseAuditWarnings release audit warnings (more than the trunk's current $trunkReleaseAuditWarnings warnings)." + -1 release audit. The applied patch generated $patchReleaseAuditWarnings release audit warnings (more than the trunk's current $OK_RELEASEAUDIT_WARNINGS warnings)." $GREP '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt > $PATCH_DIR/patchReleaseAuditProblems.txt - $GREP '\!?????' $PATCH_DIR/trunkReleaseAuditWarnings.txt > $PATCH_DIR/trunkReleaseAuditProblems.txt - echo "A diff of patched release audit warnings with trunk release audit warnings." > $PATCH_DIR/releaseAuditDiffWarnings.txt - echo "Lines that start with ????? in the release audit report indicate files that do not have an Apache license header." > $PATCH_DIR/releaseAuditDiffWarnings.txt - echo "" > $PATCH_DIR/releaseAuditDiffWarnings.txt - diff $PATCH_DIR/patchReleaseAuditProblems.txt $PATCH_DIR/trunkReleaseAuditProblems.txt >> $PATCH_DIR/releaseAuditDiffWarnings.txt - JIRA_COMMENT_FOOTER="Release audit warnings: $BUILD_URL/artifact/trunk/patchprocess/releaseAuditDiffWarnings.txt + echo "Lines that start with ????? in the release audit report indicate files that do not have an Apache license header." > $PATCH_DIR/patchReleaseAuditProblems.txt + JIRA_COMMENT_FOOTER="Release audit warnings: $BUILD_URL/artifact/trunk/patchprocess/patchReleaseAuditProblems.txt $JIRA_COMMENT_FOOTER" return 1 fi @@ -456,26 +453,22 @@ checkFindbugsWarnings () { JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/trunk/build/test/findbugs/newPatchFindbugsWarnings.html $JIRA_COMMENT_FOOTER" cp $BASEDIR/build/test/findbugs/*.xml $PATCH_DIR/patchFindbugsWarnings.xml -$FINDBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/1999" \ - $PATCH_DIR/trunkFindbugsWarnings.xml \ - $PATCH_DIR/trunkFindbugsWarnings.xml $FINDBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \ $PATCH_DIR/patchFindbugsWarnings.xml \ $PATCH_DIR/patchFindbugsWarnings.xml - $FINDBUGS_HOME/bin/computeBugHistory -output $PATCH_DIR/findbugsMerge.xml \ - $PATCH_DIR/trunkFindbugsWarnings.xml \ - $PATCH_DIR/patchFindbugsWarnings.xml - findbugsWarnings=`$FINDBUGS_HOME/bin/filterBugs -first "01/01/2000" $PATCH_DIR/findbugsMerge.xml \ + findbugsWarnings=`$FINDBUGS_HOME/bin/filterBugs -first "01/01/2000" $PATCH_DIR/patchFindbugsWarnings.xml \ $BASEDIR/build/test/findbugs/newPatchFindbugsWarnings.xml | /usr/bin/awk '{print $1}'` $FINDBUGS_HOME/bin/convertXmlToText -html \ $BASEDIR/build/test/findbugs/newPatchFindbugsWarnings.xml \ $BASEDIR/build/test/findbugs/newPatchFindbugsWarnings.html cp $BASEDIR/build/test/findbugs/newPatchFindbugsWarnings.html $PATCH_DIR/newPatchFindbugsWarnings.html cp $BASEDIR/build/test/findbugs/newPatchFindbugsWarnings.xml $PATCH_DIR/newPatchFindbugsWarnings.xml - if [[ $findbugsWarnings != 0 ]] ; then + + ### if current warnings greater than OK_FINDBUGS_WARNINGS + if [[ $findbugsWarnings > $OK_FINDBUGS_WARNINGS ]] ; then JIRA_COMMENT="$JIRA_COMMENT - -1 findbugs. The patch appears to introduce $findbugsWarnings new Findbugs warnings." + -1 findbugs. The patch appears to introduce `expr $(($findbugsWarnings-$OK_FINDBUGS_WARNINGS))` new Findbugs warnings." return 1 fi JIRA_COMMENT="$JIRA_COMMENT @@ -498,7 +491,7 @@ runCoreTests () { echo "" ### Kill any rogue build processes from the last attempt - $PS auxwww | $GREP HadoopPatchProcess | /usr/bin/nawk '{print $2}' | /usr/bin/xargs -t -I {} /usr/bin/kill -9 {} > /dev/null + $PS auxwww | $GREP HadoopPatchProcess | /usr/bin/nawk '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null PreTestTarget="" if [[ $defect == MAPREDUCE-* ]] ; then PreTestTarget="create-c++-configure" @@ -519,55 +512,6 @@ runCoreTests () { } ############################################################################### -### Tests parts of contrib specific to the eclipse files -checkJarFilesDeclaredInEclipse () { - export DECLARED_JARS=$(sed -n 's...@.*kind="lib".*path="\(.*jar\)"....@\1@p' < .eclipse.templates/.classpath) - export PRESENT_JARS=$(find build/ivy/lib/Hadoop/common/ lib/ src/test/lib/ -name '*.jar' |sort) - # When run by Hudson, consider libs from ${SUPPORT_DIR} declared - if [[ ${HUDSON} == "true" ]]; then - DECLARED_JARS="${DECLARED_JARS} $(cd "${SUPPORT_DIR}"; find lib -name '*.jar')" - fi - DECLARED_JARS=$(sed 'y/ /\n/' <<< ${DECLARED_JARS} | sort) - export ECLIPSE_DECLARED_SRC=$(sed -n 's...@.*kind="src".*path="\(.*\)"....@\1@p' < .eclipse.templates/.classpath |sort) - - if [ "${DECLARED_JARS}" != "${PRESENT_JARS}" ]; then - echo " -FAILED. Some jars are not declared in the Eclipse project. - Declared jars: ${DECLARED_JARS} - Present jars: ${PRESENT_JARS}" - return 1 - fi - for dir in $ECLIPSE_DECLARED_SRC; do - [ '!' -d $dir ] && echo " -FAILED: $dir is referenced in the Eclipse project although it doesn't exists anymore." && return 1 - done - return 0 -} - -checkEclipse () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Running Eclipse classpath verification." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - - checkJarFilesDeclaredInEclipse - if [[ $? != 0 ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - -1 Eclipse classpath. The patch causes the Eclipse classpath to differ from the contents of the lib directories." - return 1 - fi - JIRA_COMMENT="$JIRA_COMMENT - - +1 Eclipse classpath. The patch retains Eclipse classpath integrity." - return 0 -} -############################################################################### ### Run the test-contrib target runContribTests () { echo "" @@ -581,7 +525,7 @@ runContribTests () { echo "" ### Kill any rogue build processes from the last attempt - $PS -auxwww | $GREP HadoopPatchProcess | /usr/bin/nawk '{print $2}' | /usr/bin/xargs -t -I {} /usr/bin/kill -9 {} > /dev/null + $PS -auxwww | $GREP HadoopPatchProcess | /usr/bin/nawk '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" $ECLIPSE_PROPERTY $PYTHON_PROPERTY -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=yes test-contrib" $ANT_HOME/bin/ant -Dversion="${VERSION}" $ECLIPSE_PROPERTY $PYTHON_PROPERTY -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=yes test-contrib @@ -611,19 +555,19 @@ checkInjectSystemFaults () { echo "" ### Kill any rogue build processes from the last attempt - $PS auxwww | $GREP HadoopPatchProcess | /usr/bin/nawk '{print $2}' | /usr/bin/xargs -t -I {} /usr/bin/kill -9 {} > /dev/null + $PS auxwww | $GREP HadoopPatchProcess | /usr/bin/nawk '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=yes -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME -Djava5.home=$JAVA5_HOME inject-system-faults" $ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=yes -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME -Djava5.home=$JAVA5_HOME inject-system-faults if [[ $? != 0 ]] ; then JIRA_COMMENT="$JIRA_COMMENT - -1 system tests framework. The patch failed system tests framework compile." + -1 system test framework. The patch failed system test framework compile." return 1 fi JIRA_COMMENT="$JIRA_COMMENT - +1 system tests framework. The patch passed system tests framework compile." + +1 system test framework. The patch passed system test framework compile." return 0 } @@ -727,13 +671,11 @@ checkJavadocWarnings (( RESULT = RESULT + $? )) checkJavacWarnings (( RESULT = RESULT + $? )) -checkStyle -(( RESULT = RESULT + $? )) +### Checkstyle not implemented yet +#checkStyle +#(( RESULT = RESULT + $? )) checkFindbugsWarnings (( RESULT = RESULT + $? )) -##Commenting out as dependencies are now resolved using ivy -##checkEclipse -##(( RESULT = RESULT + $? )) checkReleaseAuditWarnings (( RESULT = RESULT + $? )) ### Do not call these when run by a developer