This is an automated email from the ASF dual-hosted git repository. aw pushed a commit to branch YETUS-1034-release in repository https://gitbox.apache.org/repos/asf/yetus.git
commit 977dcba5c0f2a92d4c5fbaba90799de956fc53c0 Author: Allen Wittenauer <[email protected]> AuthorDate: Tue Nov 17 12:33:27 2020 -0800 YETUS-1065. Add support for codespell (#191) Signed-off-by: Roman Shaposhnik <[email protected]> --- .codespellrc => .codespellignorelines | 7 +- .codespellrc | 4 +- asf-site-src/config.rb | 2 +- .../precommit/plugins/codespell.html.md | 52 +++++++ precommit/src/main/shell/core.d/change-analysis.sh | 59 +++++++- .../src/main/shell/test-patch-docker/Dockerfile | 2 +- precommit/src/main/shell/test-patch.d/codespell.sh | 159 +++++++++++++++++++++ website-tester.sh | 17 +++ 8 files changed, 295 insertions(+), 7 deletions(-) diff --git a/.codespellrc b/.codespellignorelines similarity index 85% copy from .codespellrc copy to .codespellignorelines index f34e714..fa991af 100644 --- a/.codespellrc +++ b/.codespellignorelines @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -[codespell] -skip = ./*.css.*,./*.js,./.git/*,./.asf.yaml,./asf-site-src/config.rb,./asf-site-src/Gemfile.lock \ No newline at end of file + rouge (~> 3.2) + rouge (3.23.0) + unless FileUtils.uptodate?(output, docs) && + FileUtils.uptodate?(output, [SHELLDOCS]) diff --git a/.codespellrc b/.codespellrc index f34e714..cb9f120 100644 --- a/.codespellrc +++ b/.codespellrc @@ -14,4 +14,6 @@ # limitations under the License. [codespell] -skip = ./*.css.*,./*.js,./.git/*,./.asf.yaml,./asf-site-src/config.rb,./asf-site-src/Gemfile.lock \ No newline at end of file +skip = *.css.*,*.js,./.git/*,./asf.yaml +ignore-regex = class\=\"nd\" + diff --git a/asf-site-src/config.rb b/asf-site-src/config.rb index b163495..033e532 100644 --- a/asf-site-src/config.rb +++ b/asf-site-src/config.rb @@ -181,7 +181,7 @@ after_configuration do # rubocop:disable Metrics/BlockLength # For Precommit we regenerate source files so they can be rendered. # we rely on a symlink. to avoid an error from the file watcher, our target - # has to be outside of hte asf-site-src directory. + # has to be outside of the asf-site-src directory. # TODO when we can, update to middleman 4 so we can use multiple source dirs # instead of symlinks FileUtils.mkdir_p 'target/in-progress/precommit/apidocs/' diff --git a/asf-site-src/source/documentation/in-progress/precommit/plugins/codespell.html.md b/asf-site-src/source/documentation/in-progress/precommit/plugins/codespell.html.md new file mode 100644 index 0000000..fba303a --- /dev/null +++ b/asf-site-src/source/documentation/in-progress/precommit/plugins/codespell.html.md @@ -0,0 +1,52 @@ +<!--- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + +# Name + +codespell + +# Category + +Test + +# Description + +Runs [codespell](https://github.com/codespell-project/codespell) on the repository. + +Due to how `codespell` executes, `./` is prefixed onto paths at runtime. This prefixing is +to be a bit more consistent with how one would run it from the command line such +that `.codespellrc` is easier to manage. + +# Environment Variables + +None + +# Options + +| Option | Notes | +|:---------|:------| +| `--codespell-exclude-lines=<file>` | File of lines that codespell should ignore (defaults to `.codespellignorelines`) | + +# Docker Notes + +None + +# Developer Notes + +None diff --git a/precommit/src/main/shell/core.d/change-analysis.sh b/precommit/src/main/shell/core.d/change-analysis.sh index f9dfc08..4456751 100755 --- a/precommit/src/main/shell/core.d/change-analysis.sh +++ b/precommit/src/main/shell/core.d/change-analysis.sh @@ -81,6 +81,62 @@ function find_changed_files popd >/dev/null || return 1 } + +## @description Determine directories with +## @description changed content. Should be used with +## @description static linters that don't care about +## @description the build system. +## @audience private +## @stability evolving +## @replaceable no +## @return None; sets ${CHANGED_DIRS} +function find_changed_dirs +{ + declare f + declare -a newarray + declare dir + + CHANGED_DIRS=() + for f in "${CHANGED_FILES[@]}"; do + dir=$(dirname "./${f}") + if [[ "${dir}" = . ]]; then + CHANGED_DIRS=('.') + continue + fi + yetus_add_array_element CHANGED_DIRS "${dir}" + done + + if [[ "${#CHANGED_DIRS[@]}" -eq 1 ]]; then + return + fi + + echo "${#CHANGED_DIRS[@]}" + + newarray=() + + for f in "${CHANGED_DIRS[@]}"; do + dir=${f%/*} + found=false + while [[ ${dir} != "." ]] && [[ ${found} = false ]]; do + if yetus_array_contains "${dir}" "${newarray[@]}"; then + found=true + continue + fi + if yetus_array_contains "${dir}" "${CHANGED_DIRS[@]}"; then + found=true + continue + fi + dir=${dir%/*} + done + + if [[ "${found}" == false ]]; then + newarray+=("${f}") + fi + done + + CHANGED_DIRS=("${newarray[@]}") +} + ## @description Apply the EXCLUDE_PATHS to CHANGED_FILES ## @audience private ## @stability stable @@ -93,7 +149,6 @@ function exclude_paths_from_changed_files declare strip declare -a a - # empty the existing list EXCLUDE_PATHS=() @@ -148,6 +203,8 @@ function exclude_paths_from_changed_files done CHANGED_FILES=("${a[@]}") + + find_changed_dirs } ## @description Check for directories to skip during diff --git a/precommit/src/main/shell/test-patch-docker/Dockerfile b/precommit/src/main/shell/test-patch-docker/Dockerfile index ae0ccb8..9014851 100644 --- a/precommit/src/main/shell/test-patch-docker/Dockerfile +++ b/precommit/src/main/shell/test-patch-docker/Dockerfile @@ -319,6 +319,7 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y \ && rm /usr/local/bin/pip /tmp/get-pip.py \ && pip3 install -v \ astroid==2.4.2 \ + git+https://github.com/codespell-project/codespell.git@8d8d4ae2bbd24a91d1a9bd8aec25c9f86724c280 \ docker-compose==1.26.2 \ pylint==2.5.3 \ yamllint==1.24.2 \ @@ -345,7 +346,6 @@ RUN echo 'gem: --no-rdoc --no-ri' >> /root/.gemrc \ ENV PATH ${PATH}:/var/tmp/.bundler-gems/bin ENV BUNDLE_PATH /var/tmp/.bundler-gems - ### # Install npm and JSHint ### diff --git a/precommit/src/main/shell/test-patch.d/codespell.sh b/precommit/src/main/shell/test-patch.d/codespell.sh new file mode 100755 index 0000000..c4325f8 --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/codespell.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# no public APIs here +# SHELLDOC-IGNORE + +add_test_type codespell + +CODESPELL_TIMER=0 +CODESPELL=${CODESPELL:-$(command -v codespell 2>/dev/null)} +CODESPELL_X_FILE=".codespellignorelines" + +function github_usage +{ + yetus_add_option "--codespell-exclude-lines=<file>" "Lines to ignore via codespell -x (default: '${CODESPELL_X_FILE}')" +} + +function github_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --codespell-exclude-lines=*) + delete_parameter "${i}" + CODESPELL_X_FILE=${i#*=} + ;; + esac + done + + CODESPELL_X_FILE=$(yetus_abs "${CODESPELL_X_FILE}") +} + +function codespell_filefilter +{ + add_test codespell +} + +function codespell_precheck +{ + if ! verify_command "codespell" "${CODESPELL}"; then + add_vote_table_v2 0 codespell "" "codespell was not available." + delete_test codespell + fi +} + +function codespell_logic +{ + declare repostatus=$1 + declare -a codespellargs + declare i + + pushd "${BASEDIR}" >/dev/null || return 1 + + # codespell will ignore skip directives if you give it + # a specific file name. so best we can do is + # use CHANGED_DIRS[@]. Will still need to filter out + # files, but this should at least cut back on the runtime + + if [[ -f "${CODESPELL_X_FILE}" ]]; then + codespellargs=(-x "${CODESPELL_X_FILE}") + fi + + for i in "${CHANGED_DIRS[@]}"; do + if [[ -f "${i}" ]]; then + # specifically add ./ because otherwise the .codespellrc file gets weird + "${CODESPELL}" \ + --disable-colors \ + --interactive 0 \ + --quiet-level 2 \ + "${codespellargs[@]}" \ + "./${i}" \ + | "${SED}" -e 's,^./,,g' \ + >> "${PATCH_DIR}/${repostatus}-codespell-tmp.txt" \ + 2>/dev/null + fi + done + + for i in "${CHANGED_FILES[@]}"; do + "${GREP}" -E "^${i}:" \ + "${PATCH_DIR}/${repostatus}-codespell-tmp.txt" \ + >> "${PATCH_DIR}/${repostatus}-codespell-result.txt" + done + + popd > /dev/null || return 1 +} + +function codespell_preapply +{ + if ! verify_needed_test codespell; then + return 0 + fi + + big_console_header "codespell plugin: ${PATCH_BRANCH}" + + start_clock + + codespell_logic branch + + # keep track of how much as elapsed for us already + CODESPELL_TIMER=$(stop_clock) + return 0 +} + +function codespell_calcdiffs +{ + error_calcdiffs "$@" +} + +function codespell_postapply +{ + declare version + + if ! verify_needed_test codespell; then + return 0 + fi + + big_console_header "codespell plugin: ${BUILDMODE}" + + start_clock + + # add our previous elapsed to our new timer + # by setting the clock back + offset_clock "${CODESPELL_TIMER}" + + codespell_logic patch + + version=$("${CODESPELL}" --version) + add_version_data codespell "${version##* v}" + + root_postlog_compare \ + codespell \ + "${PATCH_DIR}/branch-codespell-result.txt" \ + "${PATCH_DIR}/patch-codespell-result.txt" +} + +function codespell_postcompile +{ + declare repostatus=$1 + + if [[ "${repostatus}" = branch ]]; then + codespell_preapply + else + codespell_postapply + fi +} diff --git a/website-tester.sh b/website-tester.sh index 8e4367f..3139c91 100755 --- a/website-tester.sh +++ b/website-tester.sh @@ -43,6 +43,23 @@ linkchecker \ result=$? echo "::endgroup::" +echo "::group::codespell releasenotes" +codespell \ + --disable-colors \ + --interactive 0 \ + --quiet-level 2 \ + ./asf-site-src/source/downloads/releasenotes \ +| sed -e 's,^./,,g' \ + > /tmp/codespell.txt + +while read -r; do + filename=$(echo "${REPLY}" | cut -f1 -d:) + line=$(echo "${REPLY}" | cut -f2 -d:) + msg=$(echo "${REPLY}" | cut -f3 -d:) + echo "::error file=${filename},line=${line}:: codespell: ${msg}" +done < /tmp/codespell.txt +echo "::endgroup::" + # # urlname;parentname;base;result;warningstring;infostring;valid;url;line;column;name;dltime;size;checktime;cached;level;modified # in-page reference: $1
