This is an automated email from the ASF dual-hosted git repository. lewismc pushed a commit to branch NUTCH-3197 in repository https://gitbox.apache.org/repos/asf/nutch.git
commit 993abe0448b7b8736659774f0ebbddf4040740aa Author: lewismc <[email protected]> AuthorDate: Sun Aug 16 14:02:36 2026 -0700 [NUTCH-3197] Yetus: fix real precommit defects (shell, Docker, shelldocs) Child B of the Yetus master precommit initiative (follows NUTCH-3196). Fix shellcheck errors in bin/crawl and bin/nutch, add shelldocs annotations, restructure docker/Dockerfile for hadolint, exclude parse-js test sample from jshint, and waive DL3018 apk pinning via .hadolint.yaml. --- .hadolint.yaml | 4 ++++ .yetus/excludes.txt | 3 +++ README.md | 2 +- docker/Dockerfile | 23 +++++++++++------------ src/bin/crawl | 33 +++++++++++++++++++++++---------- src/bin/nutch | 25 +++++++++++++------------ 6 files changed, 55 insertions(+), 35 deletions(-) diff --git a/.hadolint.yaml b/.hadolint.yaml new file mode 100644 index 000000000..c7a9df49c --- /dev/null +++ b/.hadolint.yaml @@ -0,0 +1,4 @@ +# Hadolint config for docker/Dockerfile (NUTCH-3197). +# DL3018: do not pin floating Alpine apk versions without a deliberate bump policy. +ignored: + - DL3018 diff --git a/.yetus/excludes.txt b/.yetus/excludes.txt index 6cfe88d3c..c7d261c44 100644 --- a/.yetus/excludes.txt +++ b/.yetus/excludes.txt @@ -14,3 +14,6 @@ # # Naive Bayes training sample (deliberately messy email/OCR-like text). ^conf/naivebayes-train\.txt\.template$ +# +# parse-js sample: test fixture for link extraction, not production JS +^src/plugin/parse-js/sample/ diff --git a/README.md b/README.md index 2a4ec82fe..e2841edc1 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ test-patch --basedir=/path/to/clean/repo --build-tool=nobuild \ ``` Exclude patterns and related Yetus baselines can be added under `.yetus/` -(see `.yetus/excludes.txt`). +(see `.yetus/excludes.txt`, `.yetus/detsecrets-ignored-hashes.txt`). IDE setup --------- diff --git a/docker/Dockerfile b/docker/Dockerfile index 392dfdb86..ff94dbdcc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -30,26 +30,25 @@ LABEL org.opencontainers.image.vendor="Apache Nutch https://nutch.apache.org"; WORKDIR /root/ -# Install dependencies -RUN apk update -RUN apk --no-cache add apache-ant bash git openjdk17 +# Install dependencies and shell init for JAVA_HOME +RUN apk update && \ + apk --no-cache add apache-ant bash git openjdk17 && \ + echo 'export JAVA_HOME=/usr/lib/jvm/java-17-openjdk' >> "$HOME/.bashrc" && \ + echo 'export JAVA_HOME=/usr/lib/jvm/java-17-openjdk' >> "$HOME/.ashrc" -# Establish environment variables -RUN echo 'export JAVA_HOME=/usr/lib/jvm/java-17-openjdk' >> $HOME/.bashrc -RUN echo 'export JAVA_HOME=/usr/lib/jvm/java-17-openjdk' >> $HOME/.ashrc ENV JAVA_HOME='/usr/lib/jvm/java-17-openjdk' ENV NUTCH_HOME='/root/nutch_source/runtime/local' # Checkout and build the Nutch master branch (1.x) RUN git clone https://github.com/apache/nutch.git nutch_source && \ - cd nutch_source && \ - ant runtime && \ - rm -rf build/ && \ - rm -rf /root/.ivy2/ + cd nutch_source && \ + ant runtime && \ + rm -rf build/ && \ + rm -rf /root/.ivy2/ # Create symlinks for runtime/local/bin/nutch and runtime/local/bin/crawl -RUN ln -sf $NUTCH_HOME/bin/nutch /usr/local/bin/ -RUN ln -sf $NUTCH_HOME/bin/crawl /usr/local/bin/ +RUN ln -sf "$NUTCH_HOME/bin/nutch" /usr/local/bin/ && \ + ln -sf "$NUTCH_HOME/bin/crawl" /usr/local/bin/ RUN echo "Successfully built image, see https://s.apache.org/m5933 for guidance on running a container instance." diff --git a/src/bin/crawl b/src/bin/crawl index 409f72799..da2d48335 100755 --- a/src/bin/crawl +++ b/src/bin/crawl @@ -23,7 +23,7 @@ # # Options: # -i|--index Indexes crawl results into a configured indexer -# -D <propery>=<value> A Nutch or Hadoop property to pass to Nutch calls overwriting +# -D <property>=<value> A Nutch or Hadoop property to pass to Nutch calls overwriting # properties defined in configuration files, e.g. # increase content limit to 2MB: # -D http.content.limit=2097152 @@ -51,6 +51,8 @@ # --dedup-group <none|host|domain> Deduplication group method [default: none] # +## @audience private +## @stability stable function __to_seconds() { NUMBER=$(echo $1 | tr -dc '0-9') MODIFIER=$(echo $1 | tr -dc '[^s|h|m|d]]') @@ -73,6 +75,8 @@ function __to_seconds() { echo $SECONDS } +## @audience private +## @stability stable function __print_usage { echo "Usage: crawl [options] <crawl_dir> <num_rounds>" echo -e "" @@ -129,7 +133,7 @@ NUM_THREADS=50 SITEMAPS_FROM_HOSTDB_FREQUENCY=never DEDUP_GROUP=none -while [[ $# > 0 ]] +while [[ $# -gt 0 ]] do case $1 in -i|--index) @@ -229,9 +233,12 @@ bin="`cd "$bin"; pwd`" # determines whether mode based on presence of job file mode=local -if [ -f "${bin}"/../*nutch*.job ]; then - mode=distributed -fi +for f in "${bin}"/../*nutch*.job; do + if [ -f "$f" ]; then + mode=distributed + break + fi +done if [[ "$mode" = "local" ]]; then if [[ "$NUM_FETCHERS" -ne 1 ]]; then echo "Ignoring configured number of fetchers (--num_fetchers): a single fetcher task is used when running in local mode." @@ -247,28 +254,32 @@ commonOptions=("${HADOOP_PROPERTIES[@]}" -Dmapreduce.job.reduces=$NUM_TASKS -Dma if [ $mode = "distributed" ]; then if [ $(which hadoop | wc -l ) -eq 0 ]; then echo "Can't find Hadoop executable. Add HADOOP_COMMON_HOME/bin to the path or run in local mode." - exit -1; + exit 1; fi fi +## @audience private +## @stability stable function __bin_nutch { # run $bin/nutch, exit if exit value indicates error - echo "$bin/nutch $@" ;# echo command and arguments + echo "$bin/nutch" "$@" ;# echo command and arguments "$bin/nutch" "$@" RETCODE=$? if [ $RETCODE -ne 0 ] then echo "Error running:" - echo " $bin/nutch $@" + echo " $bin/nutch" "$@" echo "Failed with exit value $RETCODE." exit $RETCODE fi } # check if directory exists locally or on hdfs +## @audience private +## @stability stable function __directory_exists { if [[ "$mode" == local && -d "$1" ]]; then return 0 @@ -279,6 +290,8 @@ function __directory_exists { fi } +## @audience private +## @stability stable function __update_hostdb { if __directory_exists "$CRAWL_PATH"/crawldb; then echo "Updating HostDB" @@ -334,7 +347,7 @@ do generate_args=("${commonOptions[@]}" "$CRAWL_PATH"/crawldb "$CRAWL_PATH"/segments -topN $SIZE_FETCHLIST -numFetchers $NUM_FETCHERS -noFilter) fi - echo "$bin/nutch generate ${generate_args[@]}" + echo "$bin/nutch generate" "${generate_args[@]}" $bin/nutch generate "${generate_args[@]}" RETCODE=$? if [ $RETCODE -eq 0 ]; then @@ -352,7 +365,7 @@ do fi else echo "Error running:" - echo " $bin/nutch generate ${generate_args[@]}" + echo " $bin/nutch generate" "${generate_args[@]}" echo "Failed with exit value $RETCODE." exit $RETCODE fi diff --git a/src/bin/nutch b/src/bin/nutch index 87018ec32..804ce642b 100755 --- a/src/bin/nutch +++ b/src/bin/nutch @@ -147,17 +147,18 @@ fi local=true -# NUTCH_JOB -if [ -f "${NUTCH_HOME}"/*nutch*.job ]; then - local=false - for f in "$NUTCH_HOME"/*nutch*.job; do +# NUTCH_JOB +for f in "${NUTCH_HOME}"/*nutch*.job; do + if [ -f "$f" ]; then + local=false NUTCH_JOB="$f" - done - # cygwin path translation - if $cygwin; then - NUTCH_JOB="`cygpath -p -w "$NUTCH_JOB"`" + # cygwin path translation + if $cygwin; then + NUTCH_JOB="`cygpath -p -w "$NUTCH_JOB"`" + fi + break fi -fi +done JAVA="$JAVA_HOME/bin/java" JAVA_HEAP_MAX=-Xmx4096m @@ -238,7 +239,7 @@ fi # figure out which class to run if [ "$COMMAND" = "crawl" ] ; then echo "Command $COMMAND is deprecated, please use bin/crawl instead" - exit -1 + exit 1 elif [ "$COMMAND" = "inject" ] ; then CLASS=org.apache.nutch.crawl.Injector elif [ "$COMMAND" = "generate" ] ; then @@ -272,7 +273,7 @@ elif [ "$COMMAND" = "commoncrawldump" ] ; then elif [ "$COMMAND" = "solrindex" ] || [ "$COMMAND" = "solrdedup" ] || [ "$COMMAND" = "solrclean" ]; then REPLACEMENT="${COMMAND#solr}" echo "The command $COMMAND was replaced by the command $REPLACEMENT" - exit -1 + exit 1 elif [ "$COMMAND" = "index" ] ; then CLASS=org.apache.nutch.indexer.IndexingJob elif [ "$COMMAND" = "dedup" ] ; then @@ -336,7 +337,7 @@ else # check that hadoop can be found on the path if [ $(which hadoop | wc -l ) -eq 0 ]; then echo "Can't find Hadoop executable. Add HADOOP_COMMON_HOME/bin to the path or run in local mode." - exit -1; + exit 1; fi fi
