This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch YETUS-1034-release
in repository https://gitbox.apache.org/repos/asf/yetus.git
The following commit(s) were added to refs/heads/YETUS-1034-release by this
push:
new f83a855 YETUS-1070. Various github bugs/issues (#193)
f83a855 is described below
commit f83a85590f8c03432cbf5687be0b368ab04021a2
Author: Allen Wittenauer <[email protected]>
AuthorDate: Wed Nov 4 08:10:08 2020 -0800
YETUS-1070. Various github bugs/issues (#193)
---
Jenkinsfile | 3 -
.../in-progress/precommit/plugins/github.html.md | 1 +
precommit/src/main/shell/test-patch.d/github.sh | 145 +++++++++++++++++----
3 files changed, 118 insertions(+), 31 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 6a59fdc..02b7059 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -94,9 +94,6 @@ pipeline {
# enable writing back to Github
YETUS_ARGS+=(--github-token="${GITHUB_TOKEN}")
- # use emoji as vote result on GitHub
- YETUS_ARGS+=(--github-use-emoji-vote)
-
YETUS_ARGS+=(--java-home=/usr/lib/jvm/java-8-openjdk-amd64)
# enable writing back to ASF JIRA
diff --git
a/asf-site-src/source/documentation/in-progress/precommit/plugins/github.html.md
b/asf-site-src/source/documentation/in-progress/precommit/plugins/github.html.md
index 9da3ce2..6b97d69 100644
---
a/asf-site-src/source/documentation/in-progress/precommit/plugins/github.html.md
+++
b/asf-site-src/source/documentation/in-progress/precommit/plugins/github.html.md
@@ -37,6 +37,7 @@ None
| Option | Notes |
|:---------|:------|
+| `--github-annotations-limit=<int>` | Max number of GitHub Checks Annotations
to attempt |
| `--github-api-url=<url>` | REST API URL (for GitHub Enterprise) |
| `--github-base-url=<url>` | Non-REST API URL (for GitHub Enterprise) |
| `--github-repo=<repo>` | `username/repository` identifier |
diff --git a/precommit/src/main/shell/test-patch.d/github.sh
b/precommit/src/main/shell/test-patch.d/github.sh
index cdfec87..0c83079 100755
--- a/precommit/src/main/shell/test-patch.d/github.sh
+++ b/precommit/src/main/shell/test-patch.d/github.sh
@@ -35,9 +35,10 @@ GITHUB_REPO=""
# user settings
GITHUB_TOKEN="${GITHUB_TOKEN-}"
GITHUB_ISSUE=""
-GITHUB_USE_EMOJI_VOTE=false
+#GITHUB_USE_EMOJI_VOTE=false
GITHUB_STATUS_RECOVERY_COUNTER=1
GITHUB_STATUS_RECOVER_TOOL=false
+GITHUB_ANNOTATION_LIMIT=50
declare -a GITHUB_AUTH
# private globals...
@@ -45,13 +46,14 @@ GITHUB_BRIDGED=false
function github_usage
{
+ yetus_add_option "--github-annotation-limit=<int>" "The maximum number of
annotations to send to GitHub (default: '${GITHUB_ANNOTATION_LIMIT}')"
yetus_add_option "--github-api-url=<url>" "The URL of the API for github
(default: '${GITHUB_API_URL}')"
yetus_add_option "--github-base-url=<url>" "The URL of the github server
(default:'${GITHUB_BASE_URL}')"
yetus_add_option "--github-repo=<repo>" "github repo to use
(default:'${GITHUB_REPO}')"
yetus_add_option "--github-token=<token>" "The token to use to read/write to
github"
- if [[ "${GITHUB_STATUS_RECOVER_TOOL}" == false ]]; then
- yetus_add_option "--github-use-emoji-vote" "Whether to use emoji to
represent the vote result on github [default: ${GITHUB_USE_EMOJI_VOTE}]"
- fi
+ #if [[ "${GITHUB_STATUS_RECOVER_TOOL}" == false ]]; then
+ # yetus_add_option "--github-use-emoji-vote" "Whether to use emoji to
represent the vote result on github [default: ${GITHUB_USE_EMOJI_VOTE}]"
+ #fi
}
function github_parse_args
@@ -60,6 +62,10 @@ function github_parse_args
for i in "$@"; do
case ${i} in
+ --github-annotation-limit=*)
+ delete_parameter "${i}"
+ GITHUB_ANNOTATION_LIMIT=${i#*=}
+ ;;
--github-api-url=*)
delete_parameter "${i}"
GITHUB_API_URL=${i#*=}
@@ -76,10 +82,10 @@ function github_parse_args
delete_parameter "${i}"
GITHUB_TOKEN=${i#*=}
;;
- --github-use-emoji-vote)
- delete_parameter "${i}"
- GITHUB_USE_EMOJI_VOTE=true
- ;;
+ # --github-use-emoji-vote)
+ # delete_parameter "${i}"
+ # GITHUB_USE_EMOJI_VOTE=true
+ # ;;
esac
done
}
@@ -151,27 +157,65 @@ function github_breakup_url
fi
}
-## @description initialize github
-function github_initialize
+# @description guess the repo
+function github_brute_force_repo_on_remote
{
- if [[ -z "${GITHUB_REPO}" ]]; then
- GITHUB_REPO=${GITHUB_REPO_DEFAULT:-}
+ declare remote=$1
+ declare domain=${GITHUB_BASE_URL##*/}
+ declare repo
+ declare remoteurl
+
+ remoteurl=$("${GIT}" remote get-url "${remote}")
+ if [[ ${remoteurl} =~ ${domain} ]]; then
+ # chop off (protocol)(hostname)
+ repo=${remoteurl#*${domain}}
+ # chop off / or : in the front
+ repo=${repo:1}
+ # chop off ending .git
+ GITHUB_REPO=${repo%%\.git}
fi
+}
+
+## @description initialize github
+function github_initialize
+{
if [[ -n "${GITHUB_TOKEN}" ]]; then
GITHUB_AUTH=(-H "Authorization: token ${GITHUB_TOKEN}")
fi
+ if [[ -z "${GITHUB_REPO}" ]]; then
+ yetus_error "WARNING: --github-repo not autodetermined or provided. Brute
forcing."
+ pushd "${BASEDIR}" >/dev/null || return 1
+ github_brute_force_repo_on_remote origin
+ if [[ -z "${GITHUB_REPO}" ]]; then
+ while read -r; do
+ github_brute_force_repo_on_remote "${REPLY}"
+ if [[ -n "${GITHUB_REPO}" ]]; then
+ break
+ fi
+ done < <("${GIT}" remote)
+ fi
+ popd >/dev/null|| return 1
+ if [[ -n "${GITHUB_REPO}" ]]; then
+ yetus_error "WARNING: Brute force says ${GITHUB_BASE_URL}/${GITHUB_REPO}"
+ fi
+ fi
+
# if the default branch hasn't been set yet, ask GitHub
if [[ -z "${PATCH_BRANCH_DEFAULT}" && -n "${GITHUB_REPO}" && "${OFFLINE}"
== false ]]; then
- "${CURL}" --silent --fail \
+ if [[ ! -f "${PATCH_DIR}/github-repo.json" ]]; then
+ "${CURL}" --silent --fail \
-H "Accept: application/vnd.github.v3.full+json" \
"${GITHUB_AUTH[@]}" \
--output "${PATCH_DIR}/github-repo.json" \
--location \
"${GITHUB_API_URL}/repos/${GITHUB_REPO}" \
> /dev/null
- PATCH_BRANCH_DEFAULT=$("${GREP}" default_branch
"${PATCH_DIR}/github-repo.json" | head -1 | cut -d\" -f4)
+ fi
+ if [[ -f "${PATCH_DIR}/github-repo.json" ]]; then
+ PATCH_BRANCH_DEFAULT=$("${GREP}" default_branch
"${PATCH_DIR}/github-repo.json" | head -1 | cut -d\" -f4)
+ fi
fi
if [[ "${PROJECT_NAME}" == "unknown" ]]; then
@@ -604,7 +648,7 @@ function github_linecomments
fi
if [[ "${OFFLINE}" == true ]]; then
- echo "Github Plugin: Running in offline, comment skipped."
+ yetus_error "WARNING: Running offline, GitHub annotations skipped."
return 0
fi
@@ -612,6 +656,22 @@ function github_linecomments
return 0
fi
+ if [[ ${GITHUB_ANNOTATION_LIMIT} -eq 0 ]]; then
+ return 0
+ fi
+
+ ((GITHUB_ANNOTATION_LIMIT=GITHUB_ANNOTATION_LIMIT - 1))
+
+ if [[ ${GITHUB_ANNOTATION_LIMIT} -eq 0 ]]; then
+ yetus_error "WARNING: GitHub annotations limit reached."
+ return 0
+ fi
+
+ if [[ -z "${GITHUB_REPO}" ]]; then
+ yetus_error "ERROR: --github-repo is not defined."
+ return 1
+ fi
+
if [[ -z "${GITHUB_CHECK_RUN_ID}" ]]; then
if ! github_start_checkrun; then
yetus_error "ERROR: Cannot generate a Github Check Run ID"
@@ -650,18 +710,19 @@ function github_linecomments
${linehandler[@]}
${colhandler[@]}
"annotation_level": "failure",
- "message": "${newtext}"
+ "message": "${plugin}: ${newtext}"
}]
}
}
EOF
- "${CURL}" --fail -X PATCH \
+ "${CURL}" --silent --fail -X PATCH \
-H "Accept: application/vnd.github.antiope-preview+json" \
-H "Content-Type: application/json" \
"${GITHUB_AUTH[@]}" \
+ --output "${PATCH_DIR}/github-check-annotation-response.json" \
-d @"${tempfile}" \
- --silent --location \
+ --location \
"${GITHUB_API_URL}/repos/${GITHUB_REPO}/check-runs/${GITHUB_CHECK_RUN_ID}"
\
2>/dev/null
rm "${tempfile}"
@@ -678,7 +739,12 @@ function github_write_comment
declare restfile="${PATCH_DIR}/ghcomment.$$"
if [[ "${OFFLINE}" == true ]]; then
- echo "Github Plugin: Running in offline, comment skipped."
+ yetus_error "WARNING: Running offline, GitHub comment skipped."
+ return 0
+ fi
+
+ if [[ -z "${GITHUB_REPO}" ]]; then
+ yetus_error "ERROR: --github-repo is not defined."
return 0
fi
@@ -692,16 +758,17 @@ function github_write_comment
} > "${restfile}"
if [[ "${#GITHUB_AUTH[@]}" -lt 1 ]]; then
- echo "Github Plugin: no credentials provided to write a comment."
+ yetus_error "ERROR: No GitHub credentials defined."
return 0
fi
- "${CURL}" -X POST \
+ "${CURL}" --silent --fail -X POST \
-H "Accept: application/vnd.github.v3.full+json" \
-H "Content-Type: application/json" \
"${GITHUB_AUTH[@]}" \
-d @"${restfile}" \
- --silent --location \
+ --output "${PATCH_DIR}/github-comment-write-response.json" \
+ --location \
"${GITHUB_API_URL}/repos/${GITHUB_REPO}/issues/${GITHUB_ISSUE}/comments" \
>/dev/null
@@ -872,6 +939,11 @@ function github_status_write
return 0
fi
+ if [[ -z "${GITHUB_REPO}" ]]; then
+ echo "GitHub Repo not defined."
+ return 0
+ fi
+
"${CURL}" --silent --fail -X POST \
-H "Accept: application/vnd.github.v3.full+json" \
-H "Content-Type: application/json" \
@@ -879,7 +951,8 @@ function github_status_write
-d @"${filename}" \
--location \
"${GITHUB_API_URL}/repos/${GITHUB_REPO}/statuses/${GIT_BRANCH_SHA}" \
- > /dev/null
+ --output
"${PATCH_DIR}/gitub-status-response-${GITHUB_STATUS_RECOVERY_COUNTER}.json" \
+ 2>/dev/null
retval=$?
if [[ ${retval} -gt 0 ]]; then
@@ -903,6 +976,7 @@ function github_status_recovery
declare filename
declare retval=0
declare retrydir
+ declare ghr
# get the first filename
filename=$(find "${PATCH_DIR}/github-status-retry" -type f -name '1.json'
2>/dev/null)
@@ -913,10 +987,10 @@ function github_status_recovery
fi
retrydir="${filename##*/github-status-retry/}"
- GITHUB_REPO=$(echo "${retrydir}" | cut -f1-2 -d/)
+ ghr=$(echo "${retrydir}" | cut -f1-2 -d/)
+ GITHUB_REPO=${GITHUB_REPO:-${ghr}}
GIT_BRANCH_SHA=$(echo "${retrydir}" | cut -f3 -d/)
-
github_initialize
if [[ "${#GITHUB_AUTH[@]}" -lt 1 ]]; then
@@ -924,6 +998,16 @@ function github_status_recovery
return 1
fi
+ if [[ -z "${GITHUB_REPO}" ]]; then
+ yetus_error "ERROR: --github-repo is not defined."
+ return 1
+ fi
+
+ if ! github_start_checkrun; then
+ yetus_error "ERROR: Cannot generate a Github Check Run ID"
+ return 1
+ fi
+
while read -r; do
github_status_write "${REPLY}"
retval=$?
@@ -965,12 +1049,17 @@ function github_finalreport
big_console_header "Adding GitHub Statuses"
if [[ "${#GITHUB_AUTH[@]}" -lt 1 ]]; then
- echo "Github Plugin: no credentials provided to write a status."
+ yetus_error "ERROR: no credentials provided to write a status."
return 0
fi
if [[ -z "${GIT_BRANCH_SHA}" ]]; then
- echo "Unknown GIT_BRANCH_SHA defined. Skipping."
+ yetus_error "ERROR: Unknown GIT_BRANCH_SHA defined. Skipping."
+ return 0
+ fi
+
+ if [[ -z "${GITHUB_REPO}" ]]; then
+ yetus_error "ERROR: --github-repo is not defined."
return 0
fi
@@ -1088,4 +1177,4 @@ function github_finalreport
fi
((i=i+1))
done
-}
\ No newline at end of file
+}
