This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/yetus.git
The following commit(s) were added to refs/heads/main by this push:
new fc27195 YETUS-1059. Github Status Recovery Tool (#184)
fc27195 is described below
commit fc27195755bbb469c9f8c41ea29db194e017e39b
Author: Allen Wittenauer <[email protected]>
AuthorDate: Mon Nov 2 06:18:00 2020 -0800
YETUS-1059. Github Status Recovery Tool (#184)
---
Jenkinsfile | 28 ++-
.../precommit/github-status-recovery.html.md | 50 ++++
.../in-progress/precommit/index.html.md | 1 +
.../in-progress/precommit/robots/jenkins.html.md | 23 +-
precommit/src/main/shell/core.d/linecomments.sh | 132 +++++++++++
precommit/src/main/shell/github-status-recovery.sh | 188 +++++++++++++++
precommit/src/main/shell/test-patch.d/github.sh | 259 ++++++++++++++++++++-
precommit/src/main/shell/test-patch.sh | 114 +--------
8 files changed, 675 insertions(+), 120 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 0840c93..6a59fdc 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -104,9 +104,6 @@ pipeline {
YETUS_ARGS+=(--jira-password="${JIRA_PASSWORD}")
YETUS_ARGS+=(--jira-user="${JIRA_USER}")
- # disable per-line comments
- YETUS_ARGS+=(--linecomments='')
-
# pylint settings
YETUS_ARGS+=('--pylint=pylint2')
@@ -167,6 +164,31 @@ pipeline {
post {
always {
script {
+
+ // Publish status if it was missed
+ withCredentials([usernamePassword(credentialsId:
'apache-yetus-at-github.com',
+ passwordVariable: 'GITHUB_TOKEN',
+ usernameVariable: 'GITHUB_USER')]) {
+ sh '''#!/usr/bin/env bash
+
+ # enable writing back to Github
+ YETUS_ARGS+=(--github-token="${GITHUB_TOKEN}")
+
YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}")
+
+ if [[ "${USE_DEBUG_FLAG}" == true ]]; then
+ YETUS_ARGS+=("--debug")
+ fi
+
+ # run test-patch from the source tree specified up above
+
TESTPATCHBIN=${WORKSPACE}/src/precommit/src/main/shell/github-status-recovery.sh
+
+ # execute! (we are using bash instead of the
+ # bin in case the perms get messed up)
+ /usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
${EXTRA_ARGS} || true
+
+ '''
+ }
+
// Publish JUnit results
try {
junit "${env.YETUS_RELATIVE_PATCHDIR}/junit-report.xml"
diff --git
a/asf-site-src/source/documentation/in-progress/precommit/github-status-recovery.html.md
b/asf-site-src/source/documentation/in-progress/precommit/github-status-recovery.html.md
new file mode 100644
index 0000000..0cb6740
--- /dev/null
+++
b/asf-site-src/source/documentation/in-progress/precommit/github-status-recovery.html.md
@@ -0,0 +1,50 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# GitHub Status Recovery
+
+<!-- MarkdownTOC levels="1,2,3" autolink="true" indent=" " bullets="*"
bracket="round" -->
+
+* [Problem Statement](#problem-statement)
+* [Usage](#usage)
+* [Disabling Annotations](#disabling-annotations)
+
+<!-- /MarkdownTOC -->
+
+# Problem Statement
+
+For CI systems that use GitHub outside of GitHub Actions, they may make
available a GitHub Checks token.
+Unfortunately, as of this writing (2020-10-30), GitHub sets the expiry of such
a token to 1 hour.
+For some users of Apache Yetus, their precommit job may take longer than one
hour. In order to workaround
+this limitation, the `github-status-recovery` program may be used.
+
+# Usage
+
+The usage is relatively simple:
+
+```bash
+$ github-status-recovery --patch-dir=<pre-existing patch directory>
--github-token=<token>
+```
+
+If the previous run of `test-patch` failed to write the status,
`github-status-recovery` will
+re-process the saved JSON files as well as write GitHub Checks Annotations if
they exist.
+
+# Disabling Annotations
+
+If for some reason you do not wish annotations to be written, they may be
disabled with `--github-annotations=false`.
diff --git
a/asf-site-src/source/documentation/in-progress/precommit/index.html.md
b/asf-site-src/source/documentation/in-progress/precommit/index.html.md
index ed0f561..b9d3a9f 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/index.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/index.html.md
@@ -156,6 +156,7 @@ Language Support, Licensing, and more:
capacities without needing to use the full `test-patch` runtime:
* [docker-cleanup](docker-cleanup) - safe removal of Docker resources for
multi-executor CI systems
+* [github-status-recovery](github-status-recovery) - Apache Yetus status on
GitHub, even for long running jobs
* [jenkins-admin](admin) - Jenkins<->JIRA patch bridge
* [qbt](qbt) - Quality Build Tool, for branch-specific testing
* [smart-apply-patch](smart-apply-patch) - CLI manipulation and query of patch
files, PRs, and more
diff --git
a/asf-site-src/source/documentation/in-progress/precommit/robots/jenkins.html.md
b/asf-site-src/source/documentation/in-progress/precommit/robots/jenkins.html.md
index 4d81947..3de1b8c 100644
---
a/asf-site-src/source/documentation/in-progress/precommit/robots/jenkins.html.md
+++
b/asf-site-src/source/documentation/in-progress/precommit/robots/jenkins.html.md
@@ -90,8 +90,27 @@ for the optimal `test-patch` experience. Configure up to
the "Configuring the G
...
```
-Doing so will enable in many circumstances a bit more functionality, such as
-GitHub Statuses.
+Doing so will enable in many circumstances a bit more functionality, such as
GitHub Statuses.
+
+## Enabling GitHub Status Recovery
+
+If the Apache Yetus section of your job typically runs longer than 1 hour and
you use GitHub as the primary bugsystem,
+it is recommend to use the `github-status-recovery` utility in the `post`
section of your `Jenkinsfile`. For example:
+
+```groovy
+ post {
+ always {
+ script {
+ // Publish status if it was missed
+ withCredentials([usernamePassword(credentialsId: github-app',
+ passwordVariable: 'GITHUB_TOKEN',
+ usernameVariable: 'GITHUB_USER')]) {
+ sh '''github-status-recovery --patch-dir="${PATCH_DIR}""
--github-token="${GITHUB_TOKEN}"'''
+ }
+ }
+ }
+ }
+```
See also:
diff --git a/precommit/src/main/shell/core.d/linecomments.sh
b/precommit/src/main/shell/core.d/linecomments.sh
new file mode 100755
index 0000000..4ccd5e0
--- /dev/null
+++ b/precommit/src/main/shell/core.d/linecomments.sh
@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Make sure that bash version meets the pre-requisite
+
+if [[ -z "${BASH_VERSINFO[0]}" ]] \
+ || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \
+ || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then
+ echo "bash v3.2+ is required. Sorry."
+ exit 1
+fi
+
+## @description Queue up comments to write into bug systems
+## @description that have code review support, if such support
+## @description enabled/available.
+## @description File should be in the form of "file:line[:column]:comment"
+## @audience public
+## @stability evolving
+## @replaceable no
+## @param plugin
+## @param filename
+function bugsystem_linecomments_queue
+{
+ declare plugin=$1
+ declare fn=$2
+ declare line
+ declare linenum
+ declare text
+ declare columncheck
+ declare column
+ declare rol
+ declare file
+
+ if [[ -z "${BUGLINECOMMENTS}" ]]; then
+ return 0
+ fi
+
+ for line in "${VOTE_FILTER[@]}"; do
+ if [[ "${plugin}" == "${line}" ]]; then
+ return 0
+ fi
+ done
+
+ pushd "${BASEDIR}" >/dev/null || return 1
+ while read -r line; do
+ file=${line%%:*}
+
+ if [[ ! -e "${file}" ]]; then
+ continue
+ fi
+
+ rol=${line/#${file}:}
+ if [[ "${file}" =~ ^\./ ]]; then
+ file=${file:2}
+ fi
+
+ linenum=${rol%%:*}
+ rol=${rol/#${linenum}:}
+ columncheck=${rol%%:*}
+ if [[ "${columncheck}" =~ ^[0-9]+$ ]]; then
+ column=${columncheck}
+ text=${rol/#${column}:}
+ else
+ column="0"
+ text=${rol}
+ fi
+
+ echo "${file}:${linenum}:${column}:${plugin}:${text}" >>
"${PATCH_DIR}/results-full.txt"
+
+ done < "${fn}"
+
+ popd >/dev/null || return 1
+
+}
+
+## @description Write all of the bugsystem linecomments
+## @audience public
+## @stability evolving
+## @replaceable no
+function bugsystem_linecomments_trigger
+{
+ declare plugin
+ declare fn
+ declare line
+ declare linenum
+ declare text
+ declare column
+
+ if [[ ! -f "${PATCH_DIR}/results-full.txt" ]]; then
+ return 0
+ fi
+
+ # sort the file such that all files and lines are now next to each other
+ sort -k1,1 -k2,2n -k3,3n -k4,4 "${PATCH_DIR}/results-full.txt" >
"${PATCH_DIR}/linecomments-sorted.txt"
+ mv "${PATCH_DIR}/linecomments-sorted.txt" "${PATCH_DIR}/results-full.txt"
+
+ while read -r line;do
+ fn=${line%%:*}
+ rol=${line/#${fn}:}
+ linenum=${rol%%:*}
+ rol=${rol/#${linenum}:}
+ column=${rol%%:*}
+ rol=${rol/#${column}:}
+ plugin=${rol%%:*}
+ text=${rol/#${plugin}:}
+
+ for bugs in ${BUGLINECOMMENTS}; do
+ if declare -f "${bugs}_linecomments" >/dev/null;then
+ "${bugs}_linecomments" "${fn}" "${linenum}" "${column}" "${plugin}"
"${text}"
+ fi
+ done
+ done < "${PATCH_DIR}/results-full.txt"
+
+ for bugs in ${BUGLINECOMMENTS}; do
+ if declare -f "${bugs}_linecomments_end" >/dev/null;then
+ "${bugs}_linecomments_end"
+ fi
+ done
+}
\ No newline at end of file
diff --git a/precommit/src/main/shell/github-status-recovery.sh
b/precommit/src/main/shell/github-status-recovery.sh
new file mode 100755
index 0000000..97bf95a
--- /dev/null
+++ b/precommit/src/main/shell/github-status-recovery.sh
@@ -0,0 +1,188 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# no shelldocs required from this file
+# SHELLDOC-IGNORE
+
+# Make sure that bash version meets the pre-requisite
+
+if [[ -z "${BASH_VERSINFO[0]}" ]] \
+ || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \
+ || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then
+ echo "bash v3.2+ is required. Sorry."
+ exit 1
+fi
+
+this="${BASH_SOURCE-$0}"
+BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
+BINNAME=${this##*/}
+BINNAME=${BINNAME%.sh}
+#shellcheck disable=SC2034
+STARTINGDIR=$(pwd)
+#shellcheck disable=SC2034
+USER_PARAMS=("$@")
+#shellcheck disable=SC2034
+QATESTMODE=false
+#shellcheck disable=SC2034
+ISODATESTART=$(date +"%Y-%m-%dT%H:%M:%SZ")
+
+## @description import core library routines
+## @audience private
+## @stability evolving
+function import_core
+{
+ declare filename
+
+ for filename in "${BINDIR}/core.d"/*; do
+ # shellcheck disable=SC1091
+ # shellcheck source=core.d/01-common.sh
+ . "${filename}"
+ done
+}
+
+## @description import plugins then remove the stuff we don't need
+## @audience public
+## @stability stable
+## @replaceable no
+function import_including_github
+{
+ #shellcheck disable=SC2034
+ ENABLED_PLUGINS='github'
+ importplugins
+ yetus_debug "Removing BUILDTOOLS, TESTTYPES, and TESTFORMATS from installed
plug-in list"
+ #shellcheck disable=SC2034
+ BUILDTOOLS=()
+ #shellcheck disable=SC2034
+ TESTTYPES=()
+ #shellcheck disable=SC2034
+ TESTFORMATS=()
+ #shellcheck disable=SC2034
+ BUGSYSTEMS=('github')
+ #shellcheck disable=SC2034
+ BUGLINECOMMENTS='github'
+ #shellcheck disable=SC2034
+ GITHUB_STATUS_RECOVER_TOOL=true
+ GITHUB_CHECK_ANNOTATIONS=true
+}
+
+## @description Setup the default global variables
+## @audience public
+## @stability stable
+## @replaceable no
+function setup_defaults
+{
+ common_defaults
+}
+
+## @description Interpret the command line parameters
+## @audience private
+## @stability stable
+## @replaceable no
+## @param $@
+## @return May exit on failure
+function parse_args
+{
+ declare i
+ common_args "$@"
+
+ for i in "$@"; do
+ case ${i} in
+ --github-annotations=*)
+ delete_parameter "${i}"
+ GITHUB_CHECK_ANNOTATIONS="${i#*=}"
+ ;;
+ esac
+ done
+}
+
+## @description Print the usage information
+## @audience public
+## @stability stable
+## @replaceable no
+function yetus_usage
+{
+ import_including_github
+ github_usage
+
+ echo "${BINNAME} [OPTIONS]"
+
+ yetus_add_option "--github-annotations=<bool>" "Enable GitHub Checks
Annoations [default: ${GITHUB_CHECK_ANNOTATIONS}]"
+ yetus_add_option "--curl-cmd=<cmd>" "The 'curl' command to use (default
'curl')"
+ yetus_add_option "--debug" "If set, then output some extra stuff to stderr"
+ yetus_add_option "--grep-cmd=<cmd>" "The 'grep' command to use (default
'grep')"
+ yetus_add_option "--ignore-unknown-options=<bool>" "Continue despite unknown
options (default: ${IGNORE_UNKNOWN_OPTIONS})"
+ yetus_add_option "--patch-dir=<dir>" "The directory for working and output
files (default '/tmp/test-patch-${PROJECT_NAME}/pid')"
+
+ yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}"
+ yetus_reset_usage
+}
+
+## @description Large display for the user console
+## @audience public
+## @stability stable
+## @replaceable no
+## @param string
+## @return large chunk of text
+function big_console_header
+{
+ local text="$*"
+ local spacing=$(( (75+${#text}) /2 ))
+ printf '\n\n'
+ echo
"============================================================================"
+ echo
"============================================================================"
+ printf '%*s\n' ${spacing} "${text}"
+ echo
"============================================================================"
+ echo
"============================================================================"
+ printf '\n\n'
+}
+
+## @description setup the parameter tracker for param errors
+## @audience private
+## @stability evolving
+function setup_parameter_tracker
+{
+ declare i
+
+ for i in "${USER_PARAMS[@]}"; do
+ if [[ "${i}" =~ ^-- ]]; then
+ i=${i%=*}
+ PARAMETER_TRACKER+=("${i}")
+ fi
+ done
+}
+
+trap "cleanup_and_exit 1" HUP INT QUIT TERM
+
+setup_parameter_tracker
+
+import_core
+
+setup_defaults
+
+parse_args "$@"
+
+import_including_github
+
+parse_args_plugins "$@"
+
+if [[ "${#PARAMETER_TRACKER}" -gt 0 ]]; then
+ yetus_error "ERROR: Unprocessed flag(s): ${PARAMETER_TRACKER[*]}"
+ if [[ "${IGNORE_UNKNOWN_OPTIONS}" == false ]]; then
+ cleanup_and_exit 1
+ fi
+fi
+
+github_status_recovery
diff --git a/precommit/src/main/shell/test-patch.d/github.sh
b/precommit/src/main/shell/test-patch.d/github.sh
index eb01f1e..cdfec87 100755
--- a/precommit/src/main/shell/test-patch.d/github.sh
+++ b/precommit/src/main/shell/test-patch.d/github.sh
@@ -36,6 +36,8 @@ GITHUB_REPO=""
GITHUB_TOKEN="${GITHUB_TOKEN-}"
GITHUB_ISSUE=""
GITHUB_USE_EMOJI_VOTE=false
+GITHUB_STATUS_RECOVERY_COUNTER=1
+GITHUB_STATUS_RECOVER_TOOL=false
declare -a GITHUB_AUTH
# private globals...
@@ -47,7 +49,9 @@ function github_usage
yetus_add_option "--github-base-url=<url>" "The URL of the github server
(default:'${GITHUB_BASE_URL}')"
yetus_add_option "--github-repo=<repo>" "github repo to use
(default:'${GITHUB_REPO}')"
yetus_add_option "--github-token=<token>" "The token to use to read/write to
github"
- yetus_add_option "--github-use-emoji-vote" "Whether to use emoji to
represent the vote result on github [default: ${GITHUB_USE_EMOJI_VOTE}]"
+ if [[ "${GITHUB_STATUS_RECOVER_TOOL}" == false ]]; then
+ yetus_add_option "--github-use-emoji-vote" "Whether to use emoji to
represent the vote result on github [default: ${GITHUB_USE_EMOJI_VOTE}]"
+ fi
}
function github_parse_args
@@ -454,6 +458,129 @@ function github_locate_patch
esac
}
+## @description Generate a Github Check Run ID
+## @stability evolving
+## @audience private
+function github_start_checkrun
+{
+ declare tempfile="${PATCH_DIR}/ghcheckrun.$$.${RANDOM}"
+ declare output="${PATCH_DIR}/ghcheckrun.json"
+
+ if [[ -z "${GITHUB_SHA}" ]]; then
+ GITHUB_SHA=$("${GREP}" \"sha\" "${PATCH_DIR}/github-pull.json" 2>/dev/null
\
+ | head -1 \
+ | cut -f4 -d\")
+ fi
+
+ if [[ -z "${GITHUB_SHA}" ]]; then
+ return 0
+ fi
+
+ # don't need this under GHA
+ if [[ "${ROBOTTYPE}" == 'githubactions' ]]; then
+ return 0
+ fi
+
+ if [[ "${OFFLINE}" == true ]]; then
+ return 0
+ fi
+
+ if [[ "${#GITHUB_AUTH[@]}" -eq 0 ]]; then
+ return 0
+ fi
+
+ {
+ printf "{"
+ echo "\"name\":\"Apache Yetus ${ROBOTTYPE}\","
+ echo "\"head_sha\": \"${GITHUB_SHA}\","
+ echo "\"details_url\": \"${BUILD_URL}${BUILD_URL_CONSOLE}\","
+ echo "\"external_id\": \"${INSTANCE}\","
+ echo "\"status\": \"in_progress\","
+ echo "\"started_at\": \"${ISODATESTART}\""
+ # external_id instance_id?
+ # status queued, in_progress, completed
+ # started_at ISO-8601
+ # conclusion , required for status=completed, completed_at=value
+ # success, failure, neutral, cancelled, skipped, timed_out,
action_required
+ # completed_at ISO-8601
+ # output see github docs @
https://docs.github.com/en/rest/reference/checks#update-a-check-run
+ echo "}"
+ } > "${tempfile}"
+
+ "${CURL}" --silent --fail -X POST \
+ -H "Accept: application/vnd.github.antiope-preview+json" \
+ -H "Content-Type: application/json" \
+ "${GITHUB_AUTH[@]}" \
+ -d @"${tempfile}" \
+ --location \
+ "${GITHUB_API_URL}/repos/${GITHUB_REPO}/check-runs" \
+ --output "${output}" 2>/dev/null
+
+ GITHUB_CHECK_RUN_ID=$("${GREP}" \"id\" "${output}" 2>/dev/null \
+ | head -1 \
+ | cut -f2 -d: \
+ | cut -f1 -d,)
+ GITHUB_CHECK_RUN_ID=${GITHUB_CHECK_RUN_ID// /}
+}
+
+## @description Generate a Github Check Run ID
+## @stability evolving
+## @audience private
+function github_end_checkrun
+{
+ declare result=$1
+ declare tempfile="${PATCH_DIR}/ghcheckrun.$$.${RANDOM}"
+ declare output="${PATCH_DIR}/ghcheckrun-final.json"
+ declare conclusion
+
+ # don't need this under GHA
+ if [[ "${ROBOTTYPE}" == 'githubactions' ]]; then
+ return 0
+ fi
+
+ if [[ "${OFFLINE}" == true ]]; then
+ return 0
+ fi
+
+ if [[ "${#GITHUB_AUTH[@]}" -eq 0 ]]; then
+ return 0
+ fi
+
+ if [[ "${result}" -eq 0 ]]; then
+ conclusion="success"
+ else
+ conclusion="failure"
+ fi
+
+ finishdate=$(date +"%Y-%m-%dT%H:%M:%SZ")
+
+ {
+ printf "{"
+ echo "\"conclusion\":\"${conclusion}\","
+ echo "\"status\": \"completed\","
+ echo "\"completed_at\": \"${finishdate}\""
+ echo "}"
+ } > "${tempfile}"
+
+ "${CURL}" --fail --silent -X PATCH \
+ -H "Accept: application/vnd.github.antiope-preview+json" \
+ -H "Content-Type: application/json" \
+ "${GITHUB_AUTH[@]}" \
+ -d @"${tempfile}" \
+ --location \
+ "${GITHUB_API_URL}/repos/${GITHUB_REPO}/check-runs/${GITHUB_CHECK_RUN_ID}"
\
+ --output "${output}" 2>/dev/null
+ rm "${tempfile}"
+}
+
+## @description Write a Github Checks Annotation
+## @param filename
+## @param linenum
+## @param column
+## @param plugin
+## @param text
+## @stability evolving
+## @audience private
function github_linecomments
{
declare file=$1
@@ -462,6 +589,10 @@ function github_linecomments
declare plugin=$4
shift 4
declare text=$*
+ declare tempfile="${PATCH_DIR}/ghcomment.$$.${RANDOM}"
+ declare header
+ declare -a linehandler
+ declare -a colhandler
if [[ "${ROBOTTYPE}" == 'githubactions' ]]; then
if [[ -z "${column}" ]] || [[ "${column}" == 0 ]]; then
@@ -471,6 +602,69 @@ function github_linecomments
fi
return 0
fi
+
+ if [[ "${OFFLINE}" == true ]]; then
+ echo "Github Plugin: Running in offline, comment skipped."
+ return 0
+ fi
+
+ if [[ "${#GITHUB_AUTH[@]}" -eq 0 ]]; then
+ return 0
+ fi
+
+ if [[ -z "${GITHUB_CHECK_RUN_ID}" ]]; then
+ if ! github_start_checkrun; then
+ yetus_error "ERROR: Cannot generate a Github Check Run ID"
+ return 1
+ fi
+ fi
+
+ linehandler=(\"start_line\": "${linenum},")
+ linehandler+=(\"end_line\": "${linenum},")
+
+ if [[ -z "${column}" ]] || [[ "${column}" == 0 ]]; then
+ colhandler=()
+ else
+ colhandler=(\"start_column\": "${column},")
+ colhandler+=(\"end_column\": "${column},")
+ fi
+
+ newtext=$(echo "${text[*]}" | "${SED}" -e 's,\\,\\\\,g' \
+ -e 's,\",\\\",g' \
+ -e 's,$,\\r\\n,g' \
+ | tr -d '\n')
+
+ if [[ "${ROBOTTYPE}" ]]; then
+ header="Apache Yetus(${ROBOTTYPE})"
+ else
+ header="Apache Yetus"
+ fi
+
+ cat <<EOF > "${tempfile}"
+{
+ "output": {
+ "title": "${header}",
+ "summary": "Precommit Problem",
+ "annotations" : [{
+ "path": "${file}",
+ ${linehandler[@]}
+ ${colhandler[@]}
+ "annotation_level": "failure",
+ "message": "${newtext}"
+ }]
+ }
+}
+EOF
+
+ "${CURL}" --fail -X PATCH \
+ -H "Accept: application/vnd.github.antiope-preview+json" \
+ -H "Content-Type: application/json" \
+ "${GITHUB_AUTH[@]}" \
+ -d @"${tempfile}" \
+ --silent --location \
+ "${GITHUB_API_URL}/repos/${GITHUB_REPO}/check-runs/${GITHUB_CHECK_RUN_ID}"
\
+ 2>/dev/null
+ rm "${tempfile}"
}
## @description Write the contents of a file to github
@@ -663,10 +857,15 @@ function github_write_comment
## @stability evolving
## @replaceable no
## @param runresult
-function github_status_write()
+function github_status_write
{
declare filename=$1
declare retval=0
+ declare
recoverydir="${PATCH_DIR}/github-status-retry/${GITHUB_REPO}/${GIT_BRANCH_SHA}"
+
+ if [[ "${OFFLINE}" == true ]]; then
+ return 0
+ fi
if [[ "${#GITHUB_AUTH[@]}" -lt 1 ]]; then
echo "Github Plugin: no credentials provided to write a status."
@@ -684,11 +883,59 @@ function github_status_write()
retval=$?
if [[ ${retval} -gt 0 ]]; then
- echo "githubstatus-report: Failed to write status. Maybe the credential
does not have write access to repo:status."
+ yetus_error "ERROR: Failed to write github status. Token expired or
missing repo:status write?"
+ if [[ "${GITHUB_STATUS_RECOVER_TOOL}" == false ]]; then
+ mkdir -p "${recoverydir}"
+ cp -p "${tempfile}"
"${recoverydir}/${GITHUB_STATUS_RECOVERY_COUNTER}.json"
+ ((GITHUB_STATUS_RECOVERY_COUNTER=GITHUB_STATUS_RECOVERY_COUNTER+1))
+ fi
fi
return ${retval}
}
+## @description Write a github status
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param runresult
+function github_status_recovery
+{
+ declare filename
+ declare retval=0
+ declare retrydir
+
+ # get the first filename
+ filename=$(find "${PATCH_DIR}/github-status-retry" -type f -name '1.json'
2>/dev/null)
+
+ if [[ -z "${filename}" ]]; then
+ echo "No retry directory found in ${PATCH_DIR}. Maybe it was successful?"
+ return 0
+ fi
+
+ retrydir="${filename##*/github-status-retry/}"
+ GITHUB_REPO=$(echo "${retrydir}" | cut -f1-2 -d/)
+ GIT_BRANCH_SHA=$(echo "${retrydir}" | cut -f3 -d/)
+
+
+ github_initialize
+
+ if [[ "${#GITHUB_AUTH[@]}" -lt 1 ]]; then
+ echo "Github Plugin: no credentials provided to write a status."
+ return 1
+ fi
+
+ while read -r; do
+ github_status_write "${REPLY}"
+ retval=$?
+ done < <(find "${PATCH_DIR}/github-status-retry" -type f)
+
+ if [[ "${GITHUB_CHECK_ANNOTATIONS}" == true ]]; then
+ bugsystem_linecomments_trigger
+ fi
+
+ return ${retval}
+}
+
## @description Print out the finished details to the Github PR
## @audience private
## @stability evolving
@@ -711,6 +958,10 @@ function github_finalreport
declare -i i=0
declare header
+ if [[ "${OFFLINE}" == true ]]; then
+ return 0
+ fi
+
big_console_header "Adding GitHub Statuses"
if [[ "${#GITHUB_AUTH[@]}" -lt 1 ]]; then
@@ -723,6 +974,8 @@ function github_finalreport
return 0
fi
+ github_end_checkrun "${result}"
+
url=$(get_artifact_url)
if [[ "${ROBOTTYPE}" ]]; then
diff --git a/precommit/src/main/shell/test-patch.sh
b/precommit/src/main/shell/test-patch.sh
index f0cf777..93d2e12 100755
--- a/precommit/src/main/shell/test-patch.sh
+++ b/precommit/src/main/shell/test-patch.sh
@@ -68,6 +68,8 @@ function setup_defaults
common_defaults
GLOBALTIMER=$("${AWK}" 'BEGIN {srand(); print srand()}')
+ # shellcheck disable=SC2034
+ ISODATESTART=$(date +"%Y-%m-%dT%H:%M:%SZ")
set_yetus_version
@@ -2177,118 +2179,6 @@ function check_unittests
return 0
}
-## @description Queue up comments to write into bug systems
-## @description that have code review support, if such support
-## @description enabled/available.
-## @description File should be in the form of "file:line[:column]:comment"
-## @audience public
-## @stability evolving
-## @replaceable no
-## @param plugin
-## @param filename
-function bugsystem_linecomments_queue
-{
- declare plugin=$1
- declare fn=$2
- declare line
- declare linenum
- declare text
- declare columncheck
- declare column
- declare rol
- declare file
-
- if [[ -z "${BUGLINECOMMENTS}" ]]; then
- return 0
- fi
-
- for line in "${VOTE_FILTER[@]}"; do
- if [[ "${plugin}" == "${line}" ]]; then
- return 0
- fi
- done
-
- pushd "${BASEDIR}" >/dev/null || return 1
- while read -r line; do
- file=${line%%:*}
-
- if [[ ! -e "${file}" ]]; then
- continue
- fi
-
- rol=${line/#${file}:}
- if [[ "${file}" =~ ^\./ ]]; then
- file=${file:2}
- fi
-
- linenum=${rol%%:*}
- rol=${rol/#${linenum}:}
- columncheck=${rol%%:*}
- if [[ "${columncheck}" =~ ^[0-9]+$ ]]; then
- column=${columncheck}
- text=${rol/#${column}:}
- else
- column="0"
- text=${rol}
- fi
-
- echo "${file}:${linenum}:${column}:${plugin}:${text}" >>
"${PATCH_DIR}/linecomments-in.txt"
-
- done < "${fn}"
-
- popd >/dev/null || return 1
-
-}
-
-## @description Write all of the bugsystem linecomments
-## @audience public
-## @stability evolving
-## @replaceable no
-function bugsystem_linecomments_trigger
-{
- declare plugin
- declare fn
- declare line
- declare linenum
- declare text
- declare column
-
- if [[ ! -f "${PATCH_DIR}/linecomments-in.txt" ]]; then
- return 0
- fi
-
- # sort the file such that all files and lines are now next to each other
- sort -k1,1 -k2,2n -k3,3n -k4,4 "${PATCH_DIR}/linecomments-in.txt" >
"${PATCH_DIR}/linecomments-sorted.txt"
-
- while read -r line;do
- fn=${line%%:*}
- rol=${line/#${fn}:}
- linenum=${rol%%:*}
- rol=${rol/#${linenum}:}
- column=${rol%%:*}
- rol=${rol/#${column}:}
- plugin=${rol%%:*}
- text=${rol/#${plugin}:}
-
- for bugs in ${BUGLINECOMMENTS}; do
- if declare -f "${bugs}_linecomments" >/dev/null;then
- "${bugs}_linecomments" "${fn}" "${linenum}" "${column}" "${plugin}"
"${text}"
- fi
- done
- done < "${PATCH_DIR}/linecomments-sorted.txt"
-
- for bugs in ${BUGLINECOMMENTS}; do
- if declare -f "${bugs}_linecomments_end" >/dev/null;then
- "${bugs}_linecomments_end"
- fi
- done
-
- if [[ "${YETUS_SHELL_SCRIPT_DEBUG}" = true ]]; then
- yetus_debug "Keeping linecomments files for debugging"
- else
- rm "${PATCH_DIR}/linecomments-in.txt"
"${PATCH_DIR}/linecomments-sorted.txt"
- fi
-}
## @description Write the final output to the selected bug system
## @audience private
