This is an automated email from the ASF dual-hosted git repository.
lewismc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new d062e5e33 NUTCH-3197 Yetus: fix real precommit defects (shell, Docker,
shelldocs) (#958)
d062e5e33 is described below
commit d062e5e334fa3aeb7e0e893f853e805d65e68c30
Author: Lewis John McGibbney <[email protected]>
AuthorDate: Tue Sep 22 18:44:56 2026 -0700
NUTCH-3197 Yetus: fix real precommit defects (shell, Docker, shelldocs)
(#958)
* [NUTCH-3197] Yetus: fix real precommit defects (shell, Docker, shelldocs)
Child B of the Yetus master precommit initiative (follows NUTCH-3196).
Fix shellcheck errors in bin/crawl and bin/nutch, add shelldocs annotations,
restructure docker/Dockerfile for hadolint, exclude parse-js test sample
from jshint, and waive DL3018 apk pinning via .hadolint.yaml.
* [NUTCH-3197] Fix SC2006 on cygpath line introduced by nutch.job detection
refactor
Replace legacy backticks with $() on the cygpath assignment moved into the
for-loop by the SC2144 fix so Yetus patch mode no longer reports a new
shellcheck note on PR 956.
* NUTCH-3197 Yetus: fix real precommit defects (shell, Docker, shelldocs)
---
.hadolint.yaml | 4 ++++
.yetus/excludes.txt | 3 +++
README.md | 2 +-
src/bin/crawl | 34 ++++++++++++++++++++++++----------
src/bin/nutch | 26 ++++++++++++++------------
5 files changed, 46 insertions(+), 23 deletions(-)
diff --git a/.hadolint.yaml b/.hadolint.yaml
new file mode 100644
index 000000000..c7a9df49c
--- /dev/null
+++ b/.hadolint.yaml
@@ -0,0 +1,4 @@
+# Hadolint config for docker/Dockerfile (NUTCH-3197).
+# DL3018: do not pin floating Alpine apk versions without a deliberate bump
policy.
+ignored:
+ - DL3018
diff --git a/.yetus/excludes.txt b/.yetus/excludes.txt
index 6cfe88d3c..c7d261c44 100644
--- a/.yetus/excludes.txt
+++ b/.yetus/excludes.txt
@@ -14,3 +14,6 @@
#
# Naive Bayes training sample (deliberately messy email/OCR-like text).
^conf/naivebayes-train\.txt\.template$
+#
+# parse-js sample: test fixture for link extraction, not production JS
+^src/plugin/parse-js/sample/
diff --git a/README.md b/README.md
index 2a4ec82fe..e2841edc1 100644
--- a/README.md
+++ b/README.md
@@ -48,7 +48,7 @@ test-patch --basedir=/path/to/clean/repo --build-tool=nobuild
\
```
Exclude patterns and related Yetus baselines can be added under `.yetus/`
-(see `.yetus/excludes.txt`).
+(see `.yetus/excludes.txt`, `.yetus/detsecrets-ignored-hashes.txt`).
IDE setup
---------
diff --git a/src/bin/crawl b/src/bin/crawl
index 409f72799..07ac7789d 100755
--- a/src/bin/crawl
+++ b/src/bin/crawl
@@ -23,7 +23,7 @@
#
# Options:
# -i|--index Indexes crawl results into a
configured indexer
-# -D <propery>=<value> A Nutch or Hadoop property to pass
to Nutch calls overwriting
+# -D <property>=<value> A Nutch or Hadoop property to pass
to Nutch calls overwriting
# properties defined in configuration
files, e.g.
# increase content limit to 2MB:
# -D http.content.limit=2097152
@@ -51,6 +51,8 @@
# --dedup-group <none|host|domain> Deduplication group method
[default: none]
#
+## @audience private
+## @stability stable
function __to_seconds() {
NUMBER=$(echo $1 | tr -dc '0-9')
MODIFIER=$(echo $1 | tr -dc '[^s|h|m|d]]')
@@ -73,6 +75,8 @@ function __to_seconds() {
echo $SECONDS
}
+## @audience private
+## @stability stable
function __print_usage {
echo "Usage: crawl [options] <crawl_dir> <num_rounds>"
echo -e ""
@@ -129,7 +133,7 @@ NUM_THREADS=50
SITEMAPS_FROM_HOSTDB_FREQUENCY=never
DEDUP_GROUP=none
-while [[ $# > 0 ]]
+while [[ $# -gt 0 ]]
do
case $1 in
-i|--index)
@@ -229,9 +233,12 @@ bin="`cd "$bin"; pwd`"
# determines whether mode based on presence of job file
mode=local
-if [ -f "${bin}"/../*nutch*.job ]; then
- mode=distributed
-fi
+for f in "${bin}"/../*nutch*.job; do
+ if [ -f "$f" ]; then
+ mode=distributed
+ break
+ fi
+done
if [[ "$mode" = "local" ]]; then
if [[ "$NUM_FETCHERS" -ne 1 ]]; then
echo "Ignoring configured number of fetchers (--num_fetchers): a single
fetcher task is used when running in local mode."
@@ -247,28 +254,33 @@ commonOptions=("${HADOOP_PROPERTIES[@]}"
-Dmapreduce.job.reduces=$NUM_TASKS -Dma
if [ $mode = "distributed" ]; then
if [ $(which hadoop | wc -l ) -eq 0 ]; then
echo "Can't find Hadoop executable. Add HADOOP_COMMON_HOME/bin to the path
or run in local mode."
- exit -1;
+ # POSIX: 127 = utility not found (hadoop not on PATH)
+ exit 127;
fi
fi
+## @audience private
+## @stability stable
function __bin_nutch {
# run $bin/nutch, exit if exit value indicates error
- echo "$bin/nutch $@" ;# echo command and arguments
+ echo "$bin/nutch" "$@" ;# echo command and arguments
"$bin/nutch" "$@"
RETCODE=$?
if [ $RETCODE -ne 0 ]
then
echo "Error running:"
- echo " $bin/nutch $@"
+ echo " $bin/nutch" "$@"
echo "Failed with exit value $RETCODE."
exit $RETCODE
fi
}
# check if directory exists locally or on hdfs
+## @audience private
+## @stability stable
function __directory_exists {
if [[ "$mode" == local && -d "$1" ]]; then
return 0
@@ -279,6 +291,8 @@ function __directory_exists {
fi
}
+## @audience private
+## @stability stable
function __update_hostdb {
if __directory_exists "$CRAWL_PATH"/crawldb; then
echo "Updating HostDB"
@@ -334,7 +348,7 @@ do
generate_args=("${commonOptions[@]}" "$CRAWL_PATH"/crawldb
"$CRAWL_PATH"/segments -topN $SIZE_FETCHLIST -numFetchers $NUM_FETCHERS
-noFilter)
fi
- echo "$bin/nutch generate ${generate_args[@]}"
+ echo "$bin/nutch generate" "${generate_args[@]}"
$bin/nutch generate "${generate_args[@]}"
RETCODE=$?
if [ $RETCODE -eq 0 ]; then
@@ -352,7 +366,7 @@ do
fi
else
echo "Error running:"
- echo " $bin/nutch generate ${generate_args[@]}"
+ echo " $bin/nutch generate" "${generate_args[@]}"
echo "Failed with exit value $RETCODE."
exit $RETCODE
fi
diff --git a/src/bin/nutch b/src/bin/nutch
index 255a8c43e..00fca5aa4 100755
--- a/src/bin/nutch
+++ b/src/bin/nutch
@@ -147,17 +147,18 @@ fi
local=true
-# NUTCH_JOB
-if [ -f "${NUTCH_HOME}"/*nutch*.job ]; then
- local=false
- for f in "$NUTCH_HOME"/*nutch*.job; do
+# NUTCH_JOB
+for f in "${NUTCH_HOME}"/*nutch*.job; do
+ if [ -f "$f" ]; then
+ local=false
NUTCH_JOB="$f"
- done
- # cygwin path translation
- if $cygwin; then
- NUTCH_JOB="`cygpath -p -w "$NUTCH_JOB"`"
+ # cygwin path translation
+ if $cygwin; then
+ NUTCH_JOB="$(cygpath -p -w "$NUTCH_JOB")"
+ fi
+ break
fi
-fi
+done
JAVA="$JAVA_HOME/bin/java"
JAVA_HEAP_MAX=-Xmx4096m
@@ -238,7 +239,7 @@ fi
# figure out which class to run
if [ "$COMMAND" = "crawl" ] ; then
echo "Command $COMMAND is deprecated, please use bin/crawl instead"
- exit -1
+ exit 1
elif [ "$COMMAND" = "inject" ] ; then
CLASS=org.apache.nutch.crawl.Injector
elif [ "$COMMAND" = "generate" ] ; then
@@ -272,7 +273,7 @@ elif [ "$COMMAND" = "commoncrawldump" ] ; then
elif [ "$COMMAND" = "solrindex" ] || [ "$COMMAND" = "solrdedup" ] || [
"$COMMAND" = "solrclean" ]; then
REPLACEMENT="${COMMAND#solr}"
echo "The command $COMMAND was replaced by the command $REPLACEMENT"
- exit -1
+ exit 1
elif [ "$COMMAND" = "index" ] ; then
CLASS=org.apache.nutch.indexer.IndexingJob
elif [ "$COMMAND" = "dedup" ] ; then
@@ -336,7 +337,8 @@ else
# check that hadoop can be found on the path
if [ $(which hadoop | wc -l ) -eq 0 ]; then
echo "Can't find Hadoop executable. Add HADOOP_COMMON_HOME/bin to the path
or run in local mode."
- exit -1;
+ # POSIX: 127 = utility not found (hadoop not on PATH)
+ exit 127;
fi
fi
