This is an automated email from the ASF dual-hosted git repository. aw pushed a commit to branch YETUS-1034-release in repository https://gitbox.apache.org/repos/asf/yetus.git
commit 38c78a7806774a721ca5562326aabcb15cd978aa Author: Allen Wittenauer <[email protected]> AuthorDate: Mon Nov 2 06:18:00 2020 -0800 YETUS-1059. Github Status Recovery Tool (#184) --- Jenkinsfile | 28 ++- .../precommit/github-status-recovery.html.md | 50 ++++ .../in-progress/precommit/index.html.md | 1 + .../in-progress/precommit/robots/jenkins.html.md | 23 +- precommit/src/main/shell/core.d/linecomments.sh | 132 +++++++++++ precommit/src/main/shell/github-status-recovery.sh | 188 +++++++++++++++ precommit/src/main/shell/test-patch.d/github.sh | 259 ++++++++++++++++++++- precommit/src/main/shell/test-patch.sh | 114 +-------- 8 files changed, 675 insertions(+), 120 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0840c93..6a59fdc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -104,9 +104,6 @@ pipeline { YETUS_ARGS+=(--jira-password="${JIRA_PASSWORD}") YETUS_ARGS+=(--jira-user="${JIRA_USER}") - # disable per-line comments - YETUS_ARGS+=(--linecomments='') - # pylint settings YETUS_ARGS+=('--pylint=pylint2') @@ -167,6 +164,31 @@ pipeline { post { always { script { + + // Publish status if it was missed + withCredentials([usernamePassword(credentialsId: 'apache-yetus-at-github.com', + passwordVariable: 'GITHUB_TOKEN', + usernameVariable: 'GITHUB_USER')]) { + sh '''#!/usr/bin/env bash + + # enable writing back to Github + YETUS_ARGS+=(--github-token="${GITHUB_TOKEN}") + YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}") + + if [[ "${USE_DEBUG_FLAG}" == true ]]; then + YETUS_ARGS+=("--debug") + fi + + # run test-patch from the source tree specified up above + TESTPATCHBIN=${WORKSPACE}/src/precommit/src/main/shell/github-status-recovery.sh + + # execute! (we are using bash instead of the + # bin in case the perms get messed up) + /usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" ${EXTRA_ARGS} || true + + ''' + } + // Publish JUnit results try { junit "${env.YETUS_RELATIVE_PATCHDIR}/junit-report.xml" diff --git a/asf-site-src/source/documentation/in-progress/precommit/github-status-recovery.html.md b/asf-site-src/source/documentation/in-progress/precommit/github-status-recovery.html.md new file mode 100644 index 0000000..0cb6740 --- /dev/null +++ b/asf-site-src/source/documentation/in-progress/precommit/github-status-recovery.html.md @@ -0,0 +1,50 @@ +<!--- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + +# GitHub Status Recovery + +<!-- MarkdownTOC levels="1,2,3" autolink="true" indent=" " bullets="*" bracket="round" --> + +* [Problem Statement](#problem-statement) +* [Usage](#usage) +* [Disabling Annotations](#disabling-annotations) + +<!-- /MarkdownTOC --> + +# Problem Statement + +For CI systems that use GitHub outside of GitHub Actions, they may make available a GitHub Checks token. +Unfortunately, as of this writing (2020-10-30), GitHub sets the expiry of such a token to 1 hour. +For some users of Apache Yetus, their precommit job may take longer than one hour. In order to workaround +this limitation, the `github-status-recovery` program may be used. + +# Usage + +The usage is relatively simple: + +```bash +$ github-status-recovery --patch-dir=<pre-existing patch directory> --github-token=<token> +``` + +If the previous run of `test-patch` failed to write the status, `github-status-recovery` will +re-process the saved JSON files as well as write GitHub Checks Annotations if they exist. + +# Disabling Annotations + +If for some reason you do not wish annotations to be written, they may be disabled with `--github-annotations=false`. diff --git a/asf-site-src/source/documentation/in-progress/precommit/index.html.md b/asf-site-src/source/documentation/in-progress/precommit/index.html.md index ed0f561..b9d3a9f 100644 --- a/asf-site-src/source/documentation/in-progress/precommit/index.html.md +++ b/asf-site-src/source/documentation/in-progress/precommit/index.html.md @@ -156,6 +156,7 @@ Language Support, Licensing, and more: capacities without needing to use the full `test-patch` runtime: * [docker-cleanup](docker-cleanup) - safe removal of Docker resources for multi-executor CI systems +* [github-status-recovery](github-status-recovery) - Apache Yetus status on GitHub, even for long running jobs * [jenkins-admin](admin) - Jenkins<->JIRA patch bridge * [qbt](qbt) - Quality Build Tool, for branch-specific testing * [smart-apply-patch](smart-apply-patch) - CLI manipulation and query of patch files, PRs, and more diff --git a/asf-site-src/source/documentation/in-progress/precommit/robots/jenkins.html.md b/asf-site-src/source/documentation/in-progress/precommit/robots/jenkins.html.md index 4d81947..3de1b8c 100644 --- a/asf-site-src/source/documentation/in-progress/precommit/robots/jenkins.html.md +++ b/asf-site-src/source/documentation/in-progress/precommit/robots/jenkins.html.md @@ -90,8 +90,27 @@ for the optimal `test-patch` experience. Configure up to the "Configuring the G ... ``` -Doing so will enable in many circumstances a bit more functionality, such as -GitHub Statuses. +Doing so will enable in many circumstances a bit more functionality, such as GitHub Statuses. + +## Enabling GitHub Status Recovery + +If the Apache Yetus section of your job typically runs longer than 1 hour and you use GitHub as the primary bugsystem, +it is recommend to use the `github-status-recovery` utility in the `post` section of your `Jenkinsfile`. For example: + +```groovy + post { + always { + script { + // Publish status if it was missed + withCredentials([usernamePassword(credentialsId: github-app', + passwordVariable: 'GITHUB_TOKEN', + usernameVariable: 'GITHUB_USER')]) { + sh '''github-status-recovery --patch-dir="${PATCH_DIR}"" --github-token="${GITHUB_TOKEN}"''' + } + } + } + } +``` See also: diff --git a/precommit/src/main/shell/core.d/linecomments.sh b/precommit/src/main/shell/core.d/linecomments.sh new file mode 100755 index 0000000..4ccd5e0 --- /dev/null +++ b/precommit/src/main/shell/core.d/linecomments.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Make sure that bash version meets the pre-requisite + +if [[ -z "${BASH_VERSINFO[0]}" ]] \ + || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \ + || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then + echo "bash v3.2+ is required. Sorry." + exit 1 +fi + +## @description Queue up comments to write into bug systems +## @description that have code review support, if such support +## @description enabled/available. +## @description File should be in the form of "file:line[:column]:comment" +## @audience public +## @stability evolving +## @replaceable no +## @param plugin +## @param filename +function bugsystem_linecomments_queue +{ + declare plugin=$1 + declare fn=$2 + declare line + declare linenum + declare text + declare columncheck + declare column + declare rol + declare file + + if [[ -z "${BUGLINECOMMENTS}" ]]; then + return 0 + fi + + for line in "${VOTE_FILTER[@]}"; do + if [[ "${plugin}" == "${line}" ]]; then + return 0 + fi + done + + pushd "${BASEDIR}" >/dev/null || return 1 + while read -r line; do + file=${line%%:*} + + if [[ ! -e "${file}" ]]; then + continue + fi + + rol=${line/#${file}:} + if [[ "${file}" =~ ^\./ ]]; then + file=${file:2} + fi + + linenum=${rol%%:*} + rol=${rol/#${linenum}:} + columncheck=${rol%%:*} + if [[ "${columncheck}" =~ ^[0-9]+$ ]]; then + column=${columncheck} + text=${rol/#${column}:} + else + column="0" + text=${rol} + fi + + echo "${file}:${linenum}:${column}:${plugin}:${text}" >> "${PATCH_DIR}/results-full.txt" + + done < "${fn}" + + popd >/dev/null || return 1 + +} + +## @description Write all of the bugsystem linecomments +## @audience public +## @stability evolving +## @replaceable no +function bugsystem_linecomments_trigger +{ + declare plugin + declare fn + declare line + declare linenum + declare text + declare column + + if [[ ! -f "${PATCH_DIR}/results-full.txt" ]]; then + return 0 + fi + + # sort the file such that all files and lines are now next to each other + sort -k1,1 -k2,2n -k3,3n -k4,4 "${PATCH_DIR}/results-full.txt" > "${PATCH_DIR}/linecomments-sorted.txt" + mv "${PATCH_DIR}/linecomments-sorted.txt" "${PATCH_DIR}/results-full.txt" + + while read -r line;do + fn=${line%%:*} + rol=${line/#${fn}:} + linenum=${rol%%:*} + rol=${rol/#${linenum}:} + column=${rol%%:*} + rol=${rol/#${column}:} + plugin=${rol%%:*} + text=${rol/#${plugin}:} + + for bugs in ${BUGLINECOMMENTS}; do + if declare -f "${bugs}_linecomments" >/dev/null;then + "${bugs}_linecomments" "${fn}" "${linenum}" "${column}" "${plugin}" "${text}" + fi + done + done < "${PATCH_DIR}/results-full.txt" + + for bugs in ${BUGLINECOMMENTS}; do + if declare -f "${bugs}_linecomments_end" >/dev/null;then + "${bugs}_linecomments_end" + fi + done +} \ No newline at end of file diff --git a/precommit/src/main/shell/github-status-recovery.sh b/precommit/src/main/shell/github-status-recovery.sh new file mode 100755 index 0000000..97bf95a --- /dev/null +++ b/precommit/src/main/shell/github-status-recovery.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# no shelldocs required from this file +# SHELLDOC-IGNORE + +# Make sure that bash version meets the pre-requisite + +if [[ -z "${BASH_VERSINFO[0]}" ]] \ + || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \ + || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then + echo "bash v3.2+ is required. Sorry." + exit 1 +fi + +this="${BASH_SOURCE-$0}" +BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) +BINNAME=${this##*/} +BINNAME=${BINNAME%.sh} +#shellcheck disable=SC2034 +STARTINGDIR=$(pwd) +#shellcheck disable=SC2034 +USER_PARAMS=("$@") +#shellcheck disable=SC2034 +QATESTMODE=false +#shellcheck disable=SC2034 +ISODATESTART=$(date +"%Y-%m-%dT%H:%M:%SZ") + +## @description import core library routines +## @audience private +## @stability evolving +function import_core +{ + declare filename + + for filename in "${BINDIR}/core.d"/*; do + # shellcheck disable=SC1091 + # shellcheck source=core.d/01-common.sh + . "${filename}" + done +} + +## @description import plugins then remove the stuff we don't need +## @audience public +## @stability stable +## @replaceable no +function import_including_github +{ + #shellcheck disable=SC2034 + ENABLED_PLUGINS='github' + importplugins + yetus_debug "Removing BUILDTOOLS, TESTTYPES, and TESTFORMATS from installed plug-in list" + #shellcheck disable=SC2034 + BUILDTOOLS=() + #shellcheck disable=SC2034 + TESTTYPES=() + #shellcheck disable=SC2034 + TESTFORMATS=() + #shellcheck disable=SC2034 + BUGSYSTEMS=('github') + #shellcheck disable=SC2034 + BUGLINECOMMENTS='github' + #shellcheck disable=SC2034 + GITHUB_STATUS_RECOVER_TOOL=true + GITHUB_CHECK_ANNOTATIONS=true +} + +## @description Setup the default global variables +## @audience public +## @stability stable +## @replaceable no +function setup_defaults +{ + common_defaults +} + +## @description Interpret the command line parameters +## @audience private +## @stability stable +## @replaceable no +## @param $@ +## @return May exit on failure +function parse_args +{ + declare i + common_args "$@" + + for i in "$@"; do + case ${i} in + --github-annotations=*) + delete_parameter "${i}" + GITHUB_CHECK_ANNOTATIONS="${i#*=}" + ;; + esac + done +} + +## @description Print the usage information +## @audience public +## @stability stable +## @replaceable no +function yetus_usage +{ + import_including_github + github_usage + + echo "${BINNAME} [OPTIONS]" + + yetus_add_option "--github-annotations=<bool>" "Enable GitHub Checks Annoations [default: ${GITHUB_CHECK_ANNOTATIONS}]" + yetus_add_option "--curl-cmd=<cmd>" "The 'curl' command to use (default 'curl')" + yetus_add_option "--debug" "If set, then output some extra stuff to stderr" + yetus_add_option "--grep-cmd=<cmd>" "The 'grep' command to use (default 'grep')" + yetus_add_option "--ignore-unknown-options=<bool>" "Continue despite unknown options (default: ${IGNORE_UNKNOWN_OPTIONS})" + yetus_add_option "--patch-dir=<dir>" "The directory for working and output files (default '/tmp/test-patch-${PROJECT_NAME}/pid')" + + yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}" + yetus_reset_usage +} + +## @description Large display for the user console +## @audience public +## @stability stable +## @replaceable no +## @param string +## @return large chunk of text +function big_console_header +{ + local text="$*" + local spacing=$(( (75+${#text}) /2 )) + printf '\n\n' + echo "============================================================================" + echo "============================================================================" + printf '%*s\n' ${spacing} "${text}" + echo "============================================================================" + echo "============================================================================" + printf '\n\n' +} + +## @description setup the parameter tracker for param errors +## @audience private +## @stability evolving +function setup_parameter_tracker +{ + declare i + + for i in "${USER_PARAMS[@]}"; do + if [[ "${i}" =~ ^-- ]]; then + i=${i%=*} + PARAMETER_TRACKER+=("${i}") + fi + done +} + +trap "cleanup_and_exit 1" HUP INT QUIT TERM + +setup_parameter_tracker + +import_core + +setup_defaults + +parse_args "$@" + +import_including_github + +parse_args_plugins "$@" + +if [[ "${#PARAMETER_TRACKER}" -gt 0 ]]; then + yetus_error "ERROR: Unprocessed flag(s): ${PARAMETER_TRACKER[*]}" + if [[ "${IGNORE_UNKNOWN_OPTIONS}" == false ]]; then + cleanup_and_exit 1 + fi +fi + +github_status_recovery diff --git a/precommit/src/main/shell/test-patch.d/github.sh b/precommit/src/main/shell/test-patch.d/github.sh index eb01f1e..cdfec87 100755 --- a/precommit/src/main/shell/test-patch.d/github.sh +++ b/precommit/src/main/shell/test-patch.d/github.sh @@ -36,6 +36,8 @@ GITHUB_REPO="" GITHUB_TOKEN="${GITHUB_TOKEN-}" GITHUB_ISSUE="" GITHUB_USE_EMOJI_VOTE=false +GITHUB_STATUS_RECOVERY_COUNTER=1 +GITHUB_STATUS_RECOVER_TOOL=false declare -a GITHUB_AUTH # private globals... @@ -47,7 +49,9 @@ function github_usage yetus_add_option "--github-base-url=<url>" "The URL of the github server (default:'${GITHUB_BASE_URL}')" yetus_add_option "--github-repo=<repo>" "github repo to use (default:'${GITHUB_REPO}')" yetus_add_option "--github-token=<token>" "The token to use to read/write to github" - yetus_add_option "--github-use-emoji-vote" "Whether to use emoji to represent the vote result on github [default: ${GITHUB_USE_EMOJI_VOTE}]" + if [[ "${GITHUB_STATUS_RECOVER_TOOL}" == false ]]; then + yetus_add_option "--github-use-emoji-vote" "Whether to use emoji to represent the vote result on github [default: ${GITHUB_USE_EMOJI_VOTE}]" + fi } function github_parse_args @@ -454,6 +458,129 @@ function github_locate_patch esac } +## @description Generate a Github Check Run ID +## @stability evolving +## @audience private +function github_start_checkrun +{ + declare tempfile="${PATCH_DIR}/ghcheckrun.$$.${RANDOM}" + declare output="${PATCH_DIR}/ghcheckrun.json" + + if [[ -z "${GITHUB_SHA}" ]]; then + GITHUB_SHA=$("${GREP}" \"sha\" "${PATCH_DIR}/github-pull.json" 2>/dev/null \ + | head -1 \ + | cut -f4 -d\") + fi + + if [[ -z "${GITHUB_SHA}" ]]; then + return 0 + fi + + # don't need this under GHA + if [[ "${ROBOTTYPE}" == 'githubactions' ]]; then + return 0 + fi + + if [[ "${OFFLINE}" == true ]]; then + return 0 + fi + + if [[ "${#GITHUB_AUTH[@]}" -eq 0 ]]; then + return 0 + fi + + { + printf "{" + echo "\"name\":\"Apache Yetus ${ROBOTTYPE}\"," + echo "\"head_sha\": \"${GITHUB_SHA}\"," + echo "\"details_url\": \"${BUILD_URL}${BUILD_URL_CONSOLE}\"," + echo "\"external_id\": \"${INSTANCE}\"," + echo "\"status\": \"in_progress\"," + echo "\"started_at\": \"${ISODATESTART}\"" + # external_id instance_id? + # status queued, in_progress, completed + # started_at ISO-8601 + # conclusion , required for status=completed, completed_at=value + # success, failure, neutral, cancelled, skipped, timed_out, action_required + # completed_at ISO-8601 + # output see github docs @ https://docs.github.com/en/rest/reference/checks#update-a-check-run + echo "}" + } > "${tempfile}" + + "${CURL}" --silent --fail -X POST \ + -H "Accept: application/vnd.github.antiope-preview+json" \ + -H "Content-Type: application/json" \ + "${GITHUB_AUTH[@]}" \ + -d @"${tempfile}" \ + --location \ + "${GITHUB_API_URL}/repos/${GITHUB_REPO}/check-runs" \ + --output "${output}" 2>/dev/null + + GITHUB_CHECK_RUN_ID=$("${GREP}" \"id\" "${output}" 2>/dev/null \ + | head -1 \ + | cut -f2 -d: \ + | cut -f1 -d,) + GITHUB_CHECK_RUN_ID=${GITHUB_CHECK_RUN_ID// /} +} + +## @description Generate a Github Check Run ID +## @stability evolving +## @audience private +function github_end_checkrun +{ + declare result=$1 + declare tempfile="${PATCH_DIR}/ghcheckrun.$$.${RANDOM}" + declare output="${PATCH_DIR}/ghcheckrun-final.json" + declare conclusion + + # don't need this under GHA + if [[ "${ROBOTTYPE}" == 'githubactions' ]]; then + return 0 + fi + + if [[ "${OFFLINE}" == true ]]; then + return 0 + fi + + if [[ "${#GITHUB_AUTH[@]}" -eq 0 ]]; then + return 0 + fi + + if [[ "${result}" -eq 0 ]]; then + conclusion="success" + else + conclusion="failure" + fi + + finishdate=$(date +"%Y-%m-%dT%H:%M:%SZ") + + { + printf "{" + echo "\"conclusion\":\"${conclusion}\"," + echo "\"status\": \"completed\"," + echo "\"completed_at\": \"${finishdate}\"" + echo "}" + } > "${tempfile}" + + "${CURL}" --fail --silent -X PATCH \ + -H "Accept: application/vnd.github.antiope-preview+json" \ + -H "Content-Type: application/json" \ + "${GITHUB_AUTH[@]}" \ + -d @"${tempfile}" \ + --location \ + "${GITHUB_API_URL}/repos/${GITHUB_REPO}/check-runs/${GITHUB_CHECK_RUN_ID}" \ + --output "${output}" 2>/dev/null + rm "${tempfile}" +} + +## @description Write a Github Checks Annotation +## @param filename +## @param linenum +## @param column +## @param plugin +## @param text +## @stability evolving +## @audience private function github_linecomments { declare file=$1 @@ -462,6 +589,10 @@ function github_linecomments declare plugin=$4 shift 4 declare text=$* + declare tempfile="${PATCH_DIR}/ghcomment.$$.${RANDOM}" + declare header + declare -a linehandler + declare -a colhandler if [[ "${ROBOTTYPE}" == 'githubactions' ]]; then if [[ -z "${column}" ]] || [[ "${column}" == 0 ]]; then @@ -471,6 +602,69 @@ function github_linecomments fi return 0 fi + + if [[ "${OFFLINE}" == true ]]; then + echo "Github Plugin: Running in offline, comment skipped." + return 0 + fi + + if [[ "${#GITHUB_AUTH[@]}" -eq 0 ]]; then + return 0 + fi + + if [[ -z "${GITHUB_CHECK_RUN_ID}" ]]; then + if ! github_start_checkrun; then + yetus_error "ERROR: Cannot generate a Github Check Run ID" + return 1 + fi + fi + + linehandler=(\"start_line\": "${linenum},") + linehandler+=(\"end_line\": "${linenum},") + + if [[ -z "${column}" ]] || [[ "${column}" == 0 ]]; then + colhandler=() + else + colhandler=(\"start_column\": "${column},") + colhandler+=(\"end_column\": "${column},") + fi + + newtext=$(echo "${text[*]}" | "${SED}" -e 's,\\,\\\\,g' \ + -e 's,\",\\\",g' \ + -e 's,$,\\r\\n,g' \ + | tr -d '\n') + + if [[ "${ROBOTTYPE}" ]]; then + header="Apache Yetus(${ROBOTTYPE})" + else + header="Apache Yetus" + fi + + cat <<EOF > "${tempfile}" +{ + "output": { + "title": "${header}", + "summary": "Precommit Problem", + "annotations" : [{ + "path": "${file}", + ${linehandler[@]} + ${colhandler[@]} + "annotation_level": "failure", + "message": "${newtext}" + }] + } +} +EOF + + "${CURL}" --fail -X PATCH \ + -H "Accept: application/vnd.github.antiope-preview+json" \ + -H "Content-Type: application/json" \ + "${GITHUB_AUTH[@]}" \ + -d @"${tempfile}" \ + --silent --location \ + "${GITHUB_API_URL}/repos/${GITHUB_REPO}/check-runs/${GITHUB_CHECK_RUN_ID}" \ + 2>/dev/null + rm "${tempfile}" } ## @description Write the contents of a file to github @@ -663,10 +857,15 @@ function github_write_comment ## @stability evolving ## @replaceable no ## @param runresult -function github_status_write() +function github_status_write { declare filename=$1 declare retval=0 + declare recoverydir="${PATCH_DIR}/github-status-retry/${GITHUB_REPO}/${GIT_BRANCH_SHA}" + + if [[ "${OFFLINE}" == true ]]; then + return 0 + fi if [[ "${#GITHUB_AUTH[@]}" -lt 1 ]]; then echo "Github Plugin: no credentials provided to write a status." @@ -684,11 +883,59 @@ function github_status_write() retval=$? if [[ ${retval} -gt 0 ]]; then - echo "githubstatus-report: Failed to write status. Maybe the credential does not have write access to repo:status." + yetus_error "ERROR: Failed to write github status. Token expired or missing repo:status write?" + if [[ "${GITHUB_STATUS_RECOVER_TOOL}" == false ]]; then + mkdir -p "${recoverydir}" + cp -p "${tempfile}" "${recoverydir}/${GITHUB_STATUS_RECOVERY_COUNTER}.json" + ((GITHUB_STATUS_RECOVERY_COUNTER=GITHUB_STATUS_RECOVERY_COUNTER+1)) + fi fi return ${retval} } +## @description Write a github status +## @audience private +## @stability evolving +## @replaceable no +## @param runresult +function github_status_recovery +{ + declare filename + declare retval=0 + declare retrydir + + # get the first filename + filename=$(find "${PATCH_DIR}/github-status-retry" -type f -name '1.json' 2>/dev/null) + + if [[ -z "${filename}" ]]; then + echo "No retry directory found in ${PATCH_DIR}. Maybe it was successful?" + return 0 + fi + + retrydir="${filename##*/github-status-retry/}" + GITHUB_REPO=$(echo "${retrydir}" | cut -f1-2 -d/) + GIT_BRANCH_SHA=$(echo "${retrydir}" | cut -f3 -d/) + + + github_initialize + + if [[ "${#GITHUB_AUTH[@]}" -lt 1 ]]; then + echo "Github Plugin: no credentials provided to write a status." + return 1 + fi + + while read -r; do + github_status_write "${REPLY}" + retval=$? + done < <(find "${PATCH_DIR}/github-status-retry" -type f) + + if [[ "${GITHUB_CHECK_ANNOTATIONS}" == true ]]; then + bugsystem_linecomments_trigger + fi + + return ${retval} +} + ## @description Print out the finished details to the Github PR ## @audience private ## @stability evolving @@ -711,6 +958,10 @@ function github_finalreport declare -i i=0 declare header + if [[ "${OFFLINE}" == true ]]; then + return 0 + fi + big_console_header "Adding GitHub Statuses" if [[ "${#GITHUB_AUTH[@]}" -lt 1 ]]; then @@ -723,6 +974,8 @@ function github_finalreport return 0 fi + github_end_checkrun "${result}" + url=$(get_artifact_url) if [[ "${ROBOTTYPE}" ]]; then diff --git a/precommit/src/main/shell/test-patch.sh b/precommit/src/main/shell/test-patch.sh index f0cf777..93d2e12 100755 --- a/precommit/src/main/shell/test-patch.sh +++ b/precommit/src/main/shell/test-patch.sh @@ -68,6 +68,8 @@ function setup_defaults common_defaults GLOBALTIMER=$("${AWK}" 'BEGIN {srand(); print srand()}') + # shellcheck disable=SC2034 + ISODATESTART=$(date +"%Y-%m-%dT%H:%M:%SZ") set_yetus_version @@ -2177,118 +2179,6 @@ function check_unittests return 0 } -## @description Queue up comments to write into bug systems -## @description that have code review support, if such support -## @description enabled/available. -## @description File should be in the form of "file:line[:column]:comment" -## @audience public -## @stability evolving -## @replaceable no -## @param plugin -## @param filename -function bugsystem_linecomments_queue -{ - declare plugin=$1 - declare fn=$2 - declare line - declare linenum - declare text - declare columncheck - declare column - declare rol - declare file - - if [[ -z "${BUGLINECOMMENTS}" ]]; then - return 0 - fi - - for line in "${VOTE_FILTER[@]}"; do - if [[ "${plugin}" == "${line}" ]]; then - return 0 - fi - done - - pushd "${BASEDIR}" >/dev/null || return 1 - while read -r line; do - file=${line%%:*} - - if [[ ! -e "${file}" ]]; then - continue - fi - - rol=${line/#${file}:} - if [[ "${file}" =~ ^\./ ]]; then - file=${file:2} - fi - - linenum=${rol%%:*} - rol=${rol/#${linenum}:} - columncheck=${rol%%:*} - if [[ "${columncheck}" =~ ^[0-9]+$ ]]; then - column=${columncheck} - text=${rol/#${column}:} - else - column="0" - text=${rol} - fi - - echo "${file}:${linenum}:${column}:${plugin}:${text}" >> "${PATCH_DIR}/linecomments-in.txt" - - done < "${fn}" - - popd >/dev/null || return 1 - -} - -## @description Write all of the bugsystem linecomments -## @audience public -## @stability evolving -## @replaceable no -function bugsystem_linecomments_trigger -{ - declare plugin - declare fn - declare line - declare linenum - declare text - declare column - - if [[ ! -f "${PATCH_DIR}/linecomments-in.txt" ]]; then - return 0 - fi - - # sort the file such that all files and lines are now next to each other - sort -k1,1 -k2,2n -k3,3n -k4,4 "${PATCH_DIR}/linecomments-in.txt" > "${PATCH_DIR}/linecomments-sorted.txt" - - while read -r line;do - fn=${line%%:*} - rol=${line/#${fn}:} - linenum=${rol%%:*} - rol=${rol/#${linenum}:} - column=${rol%%:*} - rol=${rol/#${column}:} - plugin=${rol%%:*} - text=${rol/#${plugin}:} - - for bugs in ${BUGLINECOMMENTS}; do - if declare -f "${bugs}_linecomments" >/dev/null;then - "${bugs}_linecomments" "${fn}" "${linenum}" "${column}" "${plugin}" "${text}" - fi - done - done < "${PATCH_DIR}/linecomments-sorted.txt" - - for bugs in ${BUGLINECOMMENTS}; do - if declare -f "${bugs}_linecomments_end" >/dev/null;then - "${bugs}_linecomments_end" - fi - done - - if [[ "${YETUS_SHELL_SCRIPT_DEBUG}" = true ]]; then - yetus_debug "Keeping linecomments files for debugging" - else - rm "${PATCH_DIR}/linecomments-in.txt" "${PATCH_DIR}/linecomments-sorted.txt" - fi -} ## @description Write the final output to the selected bug system ## @audience private
