This is an automated email from the ASF dual-hosted git repository.

lewismc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git


The following commit(s) were added to refs/heads/master by this push:
     new 0713adc59 NUTCH-3224 crawl --wait hour suffix waits 120 seconds 
instead of one hour (#988)
0713adc59 is described below

commit 0713adc5965aa0a0f8f192c71aaa995309787c2f
Author: Lewis John McGibbney <[email protected]>
AuthorDate: Thu Nov 5 08:11:15 2026 -0800

    NUTCH-3224 crawl --wait hour suffix waits 120 seconds instead of one hour 
(#988)
---
 .github/workflows/master-build.yml | 30 +++++++++++++++++++++
 build.xml                          | 34 ++++++++++++++++++++++++
 default.properties                 |  4 +++
 src/bin/crawl                      |  9 +++++--
 src/test/bin/crawl_test.sh         | 53 ++++++++++++++++++++++++++++++++++++++
 5 files changed, 128 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/master-build.yml 
b/.github/workflows/master-build.yml
index c4e53ffce..a6a40fc6f 100644
--- a/.github/workflows/master-build.yml
+++ b/.github/workflows/master-build.yml
@@ -139,6 +139,36 @@ jobs:
         if: steps.filter.outputs.jenkinsfile == 'true'
         run: jenkinsfilelint --local --include 'Jenkinsfile*' 
Jenkinsfile.smoke-test-single-node-hadoop
 
+  # Shell tests. Path-filtered: runs when any bash file changes, including
+  # src/bin/crawl, src/bin/nutch, and src/test/bin.
+  bashunit:
+    strategy:
+      matrix:
+        java: ['17']
+        os: [ubuntu-24.04]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v7
+      - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d
+        id: filter
+        with:
+          filters: |
+            bashtests:
+              - 'src/bin/**'
+              - 'src/test/bin/**'
+              - '**/*.sh'
+              - 'src/plugin/parse-ext/command'
+      - name: Set up JDK ${{ matrix.java }}
+        if: steps.filter.outputs.bashtests == 'true'
+        uses: actions/setup-java@v6
+        with:
+          java-version: ${{ matrix.java }}
+          distribution: 'temurin'
+      - name: Run bashunit tests
+        if: steps.filter.outputs.bashtests == 'true'
+        run: ant test-bash -buildfile build.xml
+
   # Build verification on JDK 17 and 21 (bytecode target Java 17)
   build:
     strategy:
diff --git a/build.xml b/build.xml
index a85175909..5aca3717e 100644
--- a/build.xml
+++ b/build.xml
@@ -465,6 +465,40 @@
     </antcall>
   </target>
 
+  <!-- bashunit is a test-only download. It is not part of the JUnit suite. -->
+  <target name="bashunit-check">
+    <mkdir dir="${build.dir}"/>
+    <available file="${build.dir}/bashunit-${bashunit.version}" 
property="bashunit.present"/>
+  </target>
+
+  <target name="bashunit-download" depends="bashunit-check" 
unless="bashunit.present"
+          description="--> download the pinned bashunit release">
+    <get 
src="https://github.com/TypedDevs/bashunit/releases/download/${bashunit.version}/bashunit";
+         dest="${build.dir}/bashunit-${bashunit.version}" 
usetimestamp="false"/>
+  </target>
+
+  <target name="bashunit-verify" depends="bashunit-download">
+    <checksum file="${build.dir}/bashunit-${bashunit.version}" 
algorithm="SHA-256"
+              property="bashunit.sha256.actual"/>
+    <condition property="bashunit.checksum.ok">
+      <equals arg1="${bashunit.sha256.actual}" arg2="${bashunit.sha256}" 
casesensitive="false"/>
+    </condition>
+    <!-- Drop a bad download so the next run fetches it again. -->
+    <antcall target="bashunit-reject"/>
+    <chmod file="${build.dir}/bashunit-${bashunit.version}" perm="755"/>
+  </target>
+
+  <target name="bashunit-reject" unless="bashunit.checksum.ok">
+    <delete file="${build.dir}/bashunit-${bashunit.version}"/>
+    <fail message="bashunit ${bashunit.version} checksum mismatch (expected 
${bashunit.sha256}, got ${bashunit.sha256.actual})"/>
+  </target>
+
+  <target name="test-bash" depends="bashunit-verify" description="--> run 
bashunit tests for src/bin">
+    <exec executable="${build.dir}/bashunit-${bashunit.version}" 
failonerror="true" dir="${basedir}">
+      <arg value="src/test/bin"/>
+    </exec>
+  </target>
+
   <target name="test-core"
           depends="compile-core-test, job, jacoco-download"
           description="--> run core JUnit tests">
diff --git a/default.properties b/default.properties
index da97dae37..493d3dfa7 100644
--- a/default.properties
+++ b/default.properties
@@ -52,6 +52,10 @@ spotbugs.version=4.10.5
 # Apache RAT software distribution audit (https://creadur.apache.org/rat/)
 apache-rat.version=0.18
 
+# bashunit (https://bashunit.com/) — shell tests for src/bin
+bashunit.version=0.52.0
+bashunit.sha256=6193370a2164abcc1e64601af461ea811505c6cefdc8fa33e579a07e2d05ffe3
+
 # Forbidden APIs (https://github.com/policeman-tools/forbidden-apis)
 forbiddenapis.version=3.11
 
diff --git a/src/bin/crawl b/src/bin/crawl
index 7f60ab120..ae8f4f6ad 100755
--- a/src/bin/crawl
+++ b/src/bin/crawl
@@ -64,8 +64,7 @@ function __to_seconds() {
         SECONDS=$((10#$NUMBER * 60))
         ;;
       h|H)
-        # h means 120 seconds here, not 3600. A follow-up issue tracks the 
correction.
-        SECONDS=$((10#$NUMBER * 120))
+        SECONDS=$((10#$NUMBER * 3600))
         ;;
       d|D)
         SECONDS=$((10#$NUMBER * 86400))
@@ -121,6 +120,12 @@ function __print_usage {
   exit 1
 }
 
+# Functions above are sourced by the bashunit tests. The crawl itself runs
+# only when this file is executed.
+if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then
+  return 0
+fi
+
 # default values
 INDEXFLAG=false
 HOSTDBUPDATE=false
diff --git a/src/test/bin/crawl_test.sh b/src/test/bin/crawl_test.sh
new file mode 100755
index 000000000..9a6af45d5
--- /dev/null
+++ b/src/test/bin/crawl_test.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+## @audience private
+## @stability stable
+function set_up() {
+  local root
+  root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
+  # Path is computed from this file; ShellCheck cannot follow it.
+  # shellcheck disable=SC1091
+  source "${root}/src/bin/crawl"
+}
+
+# Rows are: wait argument, expected seconds, description shown in the report.
+# Lowercase suffixes follow the usage text. Uppercase letters are dropped by
+# the suffix filter, so the remaining digits are counted as seconds.
+## @audience private
+## @stability stable
+function provider_wait_suffixes() {
+  bashunit::data_set "1h" "3600" "lowercase h is one hour, so --wait 1h is 
3600 seconds"
+  bashunit::data_set "2h" "7200" "lowercase h is one hour, so --wait 2h is 
7200 seconds"
+  bashunit::data_set "10h" "36000" "lowercase h is one hour, so --wait 10h is 
36000 seconds"
+  bashunit::data_set "1m" "60" "lowercase m is one minute, so --wait 1m is 60 
seconds"
+  bashunit::data_set "1d" "86400" "lowercase d is one day, so --wait 1d is 
86400 seconds"
+  bashunit::data_set "1s" "1" "lowercase s is seconds, so --wait 1s is 1 
second"
+  bashunit::data_set "15" "15" "no suffix means seconds, so --wait 15 is 15 
seconds"
+  bashunit::data_set "1H" "1" "uppercase H is dropped by the suffix filter, so 
--wait 1H is 1 second"
+  bashunit::data_set "1M" "1" "uppercase M is dropped by the suffix filter, so 
--wait 1M is 1 second"
+  bashunit::data_set "1D" "1" "uppercase D is dropped by the suffix filter, so 
--wait 1D is 1 second"
+}
+
+## @audience private
+## @stability stable
+# @data_provider provider_wait_suffixes
+function test_wait_suffix_::1::_is_::2::_seconds() {
+  bashunit::set_test_title "$3"
+  assert_same "$2" "$(__to_seconds "$1")"
+}

Reply via email to