This is an automated email from the ASF dual-hosted git repository.
aengineer pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new c8d61ff HDDS-2030. Generate simplifed reports by the
dev-support/checks/*.sh scripts
c8d61ff is described below
commit c8d61ffef66ebc1d1ce2324787d0f1164d61e53f
Author: Márton Elek <e...@apache.org>
AuthorDate: Sat Aug 24 23:57:29 2019 +0200
HDDS-2030. Generate simplifed reports by the dev-support/checks/*.sh scripts
Signed-off-by: Anu Engineer <aengin...@apache.org>
Co-Authored-By: Doroszlai, Attila
<6454655+adorosz...@users.noreply.github.com>
---
hadoop-ozone/dev-support/checks/README.md | 27 +++++++++
.../dev-support/checks/_mvn_unit_report.sh | 66 ++++++++++++++++++++++
hadoop-ozone/dev-support/checks/acceptance.sh | 17 +++++-
hadoop-ozone/dev-support/checks/author.sh | 16 ++++--
hadoop-ozone/dev-support/checks/checkstyle.sh | 16 ++++--
hadoop-ozone/dev-support/checks/findbugs.sh | 20 +++----
hadoop-ozone/dev-support/checks/integration.sh | 12 ++--
hadoop-ozone/dev-support/checks/rat.sh | 11 +++-
hadoop-ozone/dev-support/checks/shellcheck.sh | 12 ++--
hadoop-ozone/dev-support/checks/unit.sh | 17 ++++--
10 files changed, 174 insertions(+), 40 deletions(-)
diff --git a/hadoop-ozone/dev-support/checks/README.md
b/hadoop-ozone/dev-support/checks/README.md
new file mode 100755
index 0000000..ba7202c
--- /dev/null
+++ b/hadoop-ozone/dev-support/checks/README.md
@@ -0,0 +1,27 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Ozone checks
+
+This directory contains a collection of easy-to-use helper scripts to execute
various type of tests on the ozone/hdds codebase.
+
+The contract of the scripts are very simple:
+
+ 1. Executing the scripts without any parameter will check the hdds/ozone
project
+ 2. Shell exit code represents the result of the check (if failed, exits with
non-zero code)
+ 3. Detailed information may be saved to the $OUTPUT_DIR (if it's not set,
root level ./target will be used).
+ 4. The standard output should contain all the log about the build AND the
results.
+ 5. The content of the $OUTPUT_DIR can be:
+ * `summary.html`/`summary.md`/`summary.txt`: contains a human readable
overview about the failed tests (used by reporting)
+ * `failures`: contains a simple number (used by reporting)
diff --git a/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
b/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
new file mode 100755
index 0000000..bb29d40
--- /dev/null
+++ b/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REPORT_DIR=${REPORT_DIR:-$PWD}
+
+## generate summary txt file
+find "." -name 'TEST*.xml' -print0 \
+ | xargs -n1 -0 "grep" -l -E "<failure|<error" \
+ | awk -F/ '{sub("'"TEST-"'",""); sub(".xml",""); print $NF}' \
+ | tee "$REPORT_DIR/summary.txt"
+
+#Copy heap dump and dump leftovers
+find "." -name "*.hprof" -exec cp {} "$REPORT_DIR/" \;
+find "." -name "*.dump" -exec cp {} "$REPORT_DIR/" \;
+
+## Add the tests where the JVM is crashed
+grep -A1 'Crashed tests' "${REPORT_DIR}/output.log" \
+ | grep -v -e 'Crashed tests' -e '--' \
+ | cut -f2- -d' ' \
+ | sort -u >> "${REPORT_DIR}/summary.txt"
+
+#Collect of all of the report failes of FAILED tests
+while IFS= read -r -d '' dir; do
+ while IFS=$'\n' read -r file; do
+ DIR_OF_TESTFILE=$(dirname "$file")
+ NAME_OF_TESTFILE=$(basename "$file")
+ NAME_OF_TEST="${NAME_OF_TESTFILE%.*}"
+ DESTDIRNAME=$(realpath --relative-to="$PWD" "$DIR_OF_TESTFILE/../..")
+ mkdir -p "$REPORT_DIR/$DESTDIRNAME"
+ #shellcheck disable=SC2086
+ cp -r "$DIR_OF_TESTFILE"/*$NAME_OF_TEST* "$REPORT_DIR/$DESTDIRNAME/"
+ done < <(grep -l -r FAILURE --include="*.txt" "$dir" | grep -v output.txt)
+done < <(find "." -name surefire-reports -print0)
+
+## generate summary markdown file
+export SUMMARY_FILE="$REPORT_DIR/summary.md"
+for TEST_RESULT_FILE in $(find "$REPORT_DIR" -name "*.txt" | grep -v output);
do
+
+ FAILURES=$(grep FAILURE "$TEST_RESULT_FILE" | grep "Tests run" | awk
'{print $18}' | sort | uniq)
+
+ for FAILURE in $FAILURES; do
+ TEST_RESULT_LOCATION="$(realpath --relative-to="$REPORT_DIR"
"$TEST_RESULT_FILE")"
+ TEST_OUTPUT_LOCATION="${TEST_RESULT_LOCATION//.txt/-output.txt/}"
+ printf " * [%s](%s) ([output](%s))\n" "$FAILURE"
"$TEST_RESULT_LOCATION" "$TEST_OUTPUT_LOCATION" >> "$SUMMARY_FILE"
+ done
+done
+
+if [ -s "$SUMMARY_FILE" ]; then
+ printf "# Failing tests: \n\n" | cat - "$SUMMARY_FILE" > temp && mv temp
"$SUMMARY_FILE"
+fi
+
+## generate counter
+wc -l "$REPORT_DIR/summary.txt" | awk '{print $1}'> "$REPORT_DIR/failures"
diff --git a/hadoop-ozone/dev-support/checks/acceptance.sh
b/hadoop-ozone/dev-support/checks/acceptance.sh
index 1e80ad4..ee03c58 100755
--- a/hadoop-ozone/dev-support/checks/acceptance.sh
+++ b/hadoop-ozone/dev-support/checks/acceptance.sh
@@ -16,7 +16,20 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/acceptance"}
+mkdir -p "$REPORT_DIR"
+
OZONE_VERSION=$(grep "<ozone.version>" "$DIR/../../pom.xml" | sed
's/<[^>]*>//g'| sed 's/^[ \t]*//')
-cd "$DIR/../../dist/target/ozone-$OZONE_VERSION/compose" || exit 1
+DIST_DIR="$DIR/../../dist/target/ozone-$OZONE_VERSION"
+
+if [ ! -d "$DIST_DIR" ]; then
+ echo "Distribution dir is missing. Doing a full build"
+ "$DIR/build.sh"
+fi
+
+cd "$DIST_DIR/compose" || exit 1
./test-all.sh
-exit $?
+RES=$?
+cp result/* "$REPORT_DIR/"
+cp "$REPORT_DIR/log.html" "$REPORT_DIR/summary.html"
+exit $RES
diff --git a/hadoop-ozone/dev-support/checks/author.sh
b/hadoop-ozone/dev-support/checks/author.sh
index f50a396..92903f9 100755
--- a/hadoop-ozone/dev-support/checks/author.sh
+++ b/hadoop-ozone/dev-support/checks/author.sh
@@ -16,12 +16,18 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
-#hide this tring to not confuse yetus
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/author"}
+mkdir -p "$REPORT_DIR"
+REPORT_FILE="$REPORT_DIR/summary.txt"
+
+#hide this string to not confuse yetus
AUTHOR="uthor"
AUTHOR="@a${AUTHOR}"
-if grep -r --include="*.java" "$AUTHOR" .; then
- exit 1
-else
- exit 0
+grep -r --include="*.java" "$AUTHOR" . | tee "$REPORT_FILE"
+
+wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures"
+
+if [[ -s "${REPORT_FILE}" ]]; then
+ exit 1
fi
diff --git a/hadoop-ozone/dev-support/checks/checkstyle.sh
b/hadoop-ozone/dev-support/checks/checkstyle.sh
index c4de528..ee11e38 100755
--- a/hadoop-ozone/dev-support/checks/checkstyle.sh
+++ b/hadoop-ozone/dev-support/checks/checkstyle.sh
@@ -16,14 +16,18 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/checkstyle"}
+mkdir -p "$REPORT_DIR"
+REPORT_FILE="$REPORT_DIR/summary.txt"
+
mvn -B -fn checkstyle:check -f pom.ozone.xml
#Print out the exact violations with parsing XML results with sed
-find "." -name checkstyle-errors.xml -print0 | xargs -0 sed '$!N;
/<file.*\n<\/file/d;P;D' | sed
'/<\/.*/d;/<checkstyle.*/d;s/<error.*line="\([[:digit:]]*\)".*message="\([^"]\+\).*/
\1: \2/;s/<file name="\([^"]*\)".*/\1/;/<\?xml.*>/d'
+find "." -name checkstyle-errors.xml -print0 | xargs -0 sed '$!N;
/<file.*\n<\/file/d;P;D' | sed
'/<\/.*/d;/<checkstyle.*/d;s/<error.*line="\([[:digit:]]*\)".*message="\([^"]\+\).*/
\1: \2/;s/<file name="\([^"]*\)".*/\1/;/<\?xml.*>/d' | tee "$REPORT_FILE"
+
+## generate counter
+wc -l "$REPORT_DIR/summary.txt" | awk '{print $1}'> "$REPORT_DIR/failures"
-violations=$(grep -r error --include checkstyle-errors.xml .| wc -l)
-if [[ $violations -gt 0 ]]; then
- echo "There are $violations checkstyle violations"
- exit 1
+if [[ -s "${REPORT_FILE}" ]]; then
+ exit 1
fi
-exit 0
diff --git a/hadoop-ozone/dev-support/checks/findbugs.sh
b/hadoop-ozone/dev-support/checks/findbugs.sh
index 545ad9f..3108bdd 100755
--- a/hadoop-ozone/dev-support/checks/findbugs.sh
+++ b/hadoop-ozone/dev-support/checks/findbugs.sh
@@ -16,21 +16,19 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
-FINDBUGS_ALL_FILE=./target/findbugs-all.txt
+mvn -B compile -fn findbugs:check -Dfindbugs.failOnError=false -f
pom.ozone.xml
-mkdir -p ./target
-rm "$FINDBUGS_ALL_FILE" || true
-touch "$FINDBUGS_ALL_FILE"
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/findbugs"}
+mkdir -p "$REPORT_DIR"
+REPORT_FILE="$REPORT_DIR/summary.txt"
-mvn -B compile -fn findbugs:check -Dfindbugs.failOnError=false -f
pom.ozone.xml
+touch "$REPORT_FILE"
-find hadoop-ozone -name findbugsXml.xml -print0 | xargs -0 -n1
convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
-find hadoop-hdds -name findbugsXml.xml -print0 | xargs -0 -n1
convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
+find hadoop-ozone -name findbugsXml.xml -print0 | xargs -0 -n1
convertXmlToText | tee -a "${REPORT_FILE}"
+find hadoop-hdds -name findbugsXml.xml -print0 | xargs -0 -n1
convertXmlToText | tee -a "${REPORT_FILE}"
-bugs=$(wc -l < "$FINDBUGS_ALL_FILE")
+wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures"
-if [[ ${bugs} -gt 0 ]]; then
+if [[ -s "${REPORT_FILE}" ]]; then
exit 1
-else
- exit 0
fi
diff --git a/hadoop-ozone/dev-support/checks/integration.sh
b/hadoop-ozone/dev-support/checks/integration.sh
index 8170c2e..ccd499d 100755
--- a/hadoop-ozone/dev-support/checks/integration.sh
+++ b/hadoop-ozone/dev-support/checks/integration.sh
@@ -20,10 +20,14 @@ export MAVEN_OPTS="-Xmx4096m"
mvn -B install -f pom.ozone.xml -DskipTests
mvn -B -fn test -f pom.ozone.xml -pl
:hadoop-ozone-integration-test,:hadoop-ozone-filesystem,:hadoop-ozone-tools \
-Dtest=\!TestMiniChaosOzoneCluster
-module_failed_tests=$(find "." -name 'TEST*.xml' -print0 \
- | xargs -0 -n1 "grep" -l -E "<failure|<error"\
- | awk -F/ '{sub("'"TEST-JUNIT_TEST_OUTPUT_DIR"'",""); sub(".xml","");
print $NF}')
-if [[ -n "${module_failed_tests}" ]] ; then
+
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/integration"}
+mkdir -p "$REPORT_DIR"
+
+# shellcheck source=hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
+source "$DIR/_mvn_unit_report.sh"
+
+if [[ -s "$REPORT_DIR/summary.txt" ]] ; then
exit 1
fi
exit 0
diff --git a/hadoop-ozone/dev-support/checks/rat.sh
b/hadoop-ozone/dev-support/checks/rat.sh
index 480e4d3..464d636 100755
--- a/hadoop-ozone/dev-support/checks/rat.sh
+++ b/hadoop-ozone/dev-support/checks/rat.sh
@@ -16,9 +16,10 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
-mkdir -p target
-REPORT_FILE="$DIR/../../../target/rat-aggregated.txt"
-mkdir -p "$(dirname "$REPORT_FILE")"
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/rat"}
+mkdir -p "$REPORT_DIR"
+
+REPORT_FILE="$REPORT_DIR/summary.txt"
cd hadoop-hdds || exit 1
mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check
@@ -26,7 +27,11 @@ cd ../hadoop-ozone || exit 1
mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check
cd "$DIR/../../.." || exit 1
+
grep -r --include=rat.txt "!????" hadoop-hdds hadoop-ozone | tee "$REPORT_FILE"
+
+wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures"
+
if [[ -s "${REPORT_FILE}" ]]; then
exit 1
fi
diff --git a/hadoop-ozone/dev-support/checks/shellcheck.sh
b/hadoop-ozone/dev-support/checks/shellcheck.sh
index 637a4f8..2b67118 100755
--- a/hadoop-ozone/dev-support/checks/shellcheck.sh
+++ b/hadoop-ozone/dev-support/checks/shellcheck.sh
@@ -16,8 +16,10 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
-OUTPUT_FILE="$DIR/../../../target/shell-problems.txt"
-mkdir -p "$(dirname "$OUTPUT_FILE")"
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/shellcheck"}
+mkdir -p "$REPORT_DIR"
+REPORT_FILE="$REPORT_DIR/summary.txt"
+
echo "" > "$OUTPUT_FILE"
if [[ "$(uname -s)" = "Darwin" ]]; then
find hadoop-hdds hadoop-ozone -type f -perm '-500'
@@ -26,8 +28,10 @@ else
fi \
| grep -v -e target/ -e node_modules/ -e '\.\(ico\|py\|yml\)$' \
| xargs -n1 shellcheck \
- | tee "$OUTPUT_FILE"
+ | tee "$REPORT_FILE"
+
+wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures"
-if [ "$(cat "$OUTPUT_FILE")" ]; then
+if [[ -s "${REPORT_FILE}" ]]; then
exit 1
fi
diff --git a/hadoop-ozone/dev-support/checks/unit.sh
b/hadoop-ozone/dev-support/checks/unit.sh
index 473f714..9429026 100755
--- a/hadoop-ozone/dev-support/checks/unit.sh
+++ b/hadoop-ozone/dev-support/checks/unit.sh
@@ -13,12 +13,19 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
+
export MAVEN_OPTS="-Xmx4096m"
-mvn -fn test -f pom.ozone.xml -pl
\!:hadoop-ozone-integration-test,\!:hadoop-ozone-filesystem,\!:hadoop-ozone-tools
-module_failed_tests=$(find "." -name 'TEST*.xml' -print0 \
- | xargs -n1 -0 "grep" -l -E "<failure|<error"\
- | awk -F/ '{sub("'"TEST-JUNIT_TEST_OUTPUT_DIR"'",""); sub(".xml","");
print $NF}')
-if [[ -n "${module_failed_tests}" ]] ; then
+mvn -B -fn test -f pom.ozone.xml -pl
\!:hadoop-ozone-integration-test,\!:hadoop-ozone-filesystem,\!:hadoop-ozone-tools
+
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/unit"}
+mkdir -p "$REPORT_DIR"
+
+# shellcheck source=hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
+source "$DIR/_mvn_unit_report.sh"
+
+if [[ -s "$REPORT_DIR/summary.txt" ]] ; then
exit 1
fi
exit 0
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
