This is an automated email from the ASF dual-hosted git repository.
lewismc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new 6990f21c9 NUTCH-3194 Add Jenkinsfile for single-node Hadoop smoke test
job (#946)
6990f21c9 is described below
commit 6990f21c974c2b495633ebb9008c97c3c0b65a9e
Author: Lewis John McGibbney <[email protected]>
AuthorDate: Wed Aug 5 15:16:02 2026 -0700
NUTCH-3194 Add Jenkinsfile for single-node Hadoop smoke test job (#946)
---
.github/workflows/master-build.yml | 25 +++
Jenkinsfile.smoke-test-single-node-hadoop | 321 ++++++++++++++++++++++++++++++
2 files changed, 346 insertions(+)
diff --git a/.github/workflows/master-build.yml
b/.github/workflows/master-build.yml
index 74072f591..02d6faf9a 100644
--- a/.github/workflows/master-build.yml
+++ b/.github/workflows/master-build.yml
@@ -114,6 +114,31 @@ jobs:
if: steps.filter.outputs.openapi == 'true'
run: ./node_modules/.bin/lint-openapi openapi.yaml
+ # Declarative Pipeline syntax check (jenkinsfilelint --local; Docker
required).
+ # Path-filtered: only runs when Jenkinsfile* changes. Local mode validates
+ # vanilla Declarative syntax, not every plugin on ci-builds.apache.org.
+ jenkinsfile-lint:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v7
+ - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706
+ id: filter
+ with:
+ filters: |
+ jenkinsfile:
+ - 'Jenkinsfile*'
+ - name: Set up Python
+ if: steps.filter.outputs.jenkinsfile == 'true'
+ uses: actions/setup-python@v7
+ with:
+ python-version: '3.13'
+ - name: Install jenkinsfilelint
+ if: steps.filter.outputs.jenkinsfile == 'true'
+ run: pip install jenkinsfilelint==1.5.1
+ - name: Validate Jenkinsfile Declarative Pipeline syntax
+ if: steps.filter.outputs.jenkinsfile == 'true'
+ run: jenkinsfilelint --local --include 'Jenkinsfile*'
Jenkinsfile.smoke-test-single-node-hadoop
+
# Build verification on JDK 17 and 21 (bytecode target Java 17)
build:
strategy:
diff --git a/Jenkinsfile.smoke-test-single-node-hadoop
b/Jenkinsfile.smoke-test-single-node-hadoop
new file mode 100644
index 000000000..6c208a498
--- /dev/null
+++ b/Jenkinsfile.smoke-test-single-node-hadoop
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Nutch Smoke Test — single-node (pseudo-distributed) Hadoop cluster.
+//
+// Trigger with "Build with Parameters":
+// GIT_REF — ref to build from https://github.com/apache/nutch.git (default:
master).
+// Branches: NUTCH-3026, master
+// Tags: release-1.23 (preferred if the same name exists as a branch)
+// PRs: pr/866 (fetches pull/866/head; do not pass a bare
number)
+// Blank/whitespace GIT_REF is treated as master.
+// EMAIL_RECIPIENT — optional job-result notification address
+// HADOOP_VERSION — Hadoop tarball version to download (default: 3.4.2)
+//
+// Paste this script into the ASF Jenkins job configure page when updating the
live job.
+// Landing Jenkinsfile(s) in the apache/nutch git repo is tracked separately.
+
+pipeline {
+ agent { label 'ubuntu' }
+ parameters {
+ string(
+ name: 'GIT_REF',
+ defaultValue: 'master',
+ description: 'Git ref to build from
https://github.com/apache/nutch.git. Examples: master (default;
blank/whitespace also uses master); branch NUTCH-3026; tag release-1.23; PR
pr/866 (not a bare PR number). If a name exists as both a tag and a branch, the
tag is used.'
+ )
+ string(name: 'EMAIL_RECIPIENT', defaultValue: '', description: 'Email
address for job result notification (optional)')
+ string(name: 'HADOOP_VERSION', defaultValue: '3.5.0', description:
'Hadoop version to download (e.g., 3.X.X)')
+ }
+ environment {
+ HADOOP_HOME = "${WORKSPACE}/hadoop-${params.HADOOP_VERSION}"
+ NUTCH_HOME = "${WORKSPACE}/nutch/runtime/deploy"
+ JAVA_HOME = "/home/jenkins/tools/java/latest17"
+ CACHE_DIR = "${WORKSPACE}/cache/hadoop"
+ HADOOP_TAR = "hadoop-${params.HADOOP_VERSION}.tar.gz"
+ HADOOP_CHECKSUM = "hadoop-${params.HADOOP_VERSION}.tar.gz.sha512"
+ HADOOP_DIR = "hadoop-${params.HADOOP_VERSION}"
+ DOWNLOAD_URL =
"https://dlcdn.apache.org/hadoop/common/hadoop-${params.HADOOP_VERSION}";
+ PATH =
"${WORKSPACE}/hadoop-${params.HADOOP_VERSION}/bin:${WORKSPACE}/hadoop-${params.HADOOP_VERSION}/sbin:${WORKSPACE}/nutch/runtime/deploy/bin:${env.PATH}"
+ }
+ stages {
+ stage('Resolve GIT_REF') {
+ steps {
+ script {
+ def gitRef = (params.GIT_REF ?: '').trim()
+ if (!gitRef) {
+ gitRef = 'master'
+ }
+ if (!(gitRef ==~ /^(pr\/[0-9]+|[A-Za-z0-9._\/-]+)$/)) {
+ error("Invalid GIT_REF '${gitRef}'. Use a branch or
tag name (e.g. master, NUTCH-3026, release-1.23) or pr/<id> (e.g. pr/866).")
+ }
+ env.RESOLVED_GIT_REF = gitRef
+ currentBuild.displayName = "#${env.BUILD_NUMBER} ·
${gitRef}"
+ currentBuild.description = "Git Ref: ${gitRef}"
+ echo "Resolved GIT_REF=${gitRef}"
+ }
+ }
+ }
+ stage('Checkout nutch-test-single-node-cluster') {
+ steps {
+ git url:
'https://github.com/sebastian-nagel/nutch-test-single-node-cluster.git',
+ branch: 'main'
+ }
+ }
+ stage("Restore Hadoop from Cache") {
+ steps {
+ sh """
+ echo "Creating cache directory if it doesn't exist..."
+ mkdir -p ${CACHE_DIR}
+
+ # Restore cached files
+ if [ -f "${CACHE_DIR}/${HADOOP_TAR}" ]; then
+ echo "Restoring Hadoop tarball from cache..."
+ cp "${CACHE_DIR}/${HADOOP_TAR}" .
+ fi
+
+ if [ -f "${CACHE_DIR}/${HADOOP_CHECKSUM}" ]; then
+ echo "Restoring checksum from cache..."
+ cp "${CACHE_DIR}/${HADOOP_CHECKSUM}" .
+ fi
+
+ if [ -d "${CACHE_DIR}/${HADOOP_DIR}" ]; then
+ echo "Restoring Hadoop directory from cache..."
+ cp -r "${CACHE_DIR}/${HADOOP_DIR}" .
+ fi
+ """
+ }
+ }
+ stage("Download and Install Hadoop") {
+ steps {
+ script {
+ def stageName = "Download and Install Hadoop
${params.HADOOP_VERSION}"
+ stage(stageName) {
+ sh """
+ # Download tarball if it doesn't exist
+ if [ ! -f "${HADOOP_TAR}" ]; then
+ echo "Downloading Hadoop
${params.HADOOP_VERSION}..."
+ wget ${DOWNLOAD_URL}/${HADOOP_TAR}
+
+ echo "Downloading checksum file..."
+ wget ${DOWNLOAD_URL}/${HADOOP_CHECKSUM}
+ else
+ echo "Hadoop tarball already exists (restored
from cache)"
+
+ if [ ! -f "${HADOOP_CHECKSUM}" ]; then
+ echo "Downloading checksum file..."
+ wget ${DOWNLOAD_URL}/${HADOOP_CHECKSUM}
+ fi
+ fi
+
+ # Verify checksum
+ echo "Verifying SHA-512 checksum..."
+ if sha512sum -c ${HADOOP_CHECKSUM}; then
+ echo "Checksum verification PASSED"
+ else
+ echo "Checksum verification FAILED"
+ echo "Removing corrupted tarball..."
+ rm -f ${HADOOP_TAR}
+ # Also remove from cache
+ rm -f ${CACHE_DIR}/${HADOOP_TAR}
+ exit 1
+ fi
+
+ # Update cache with latest files
+ echo "Updating cache..."
+ cp -u ${HADOOP_TAR} ${CACHE_DIR}/
+ cp -u ${HADOOP_CHECKSUM} ${CACHE_DIR}/
+
+ # Update Hadoop configuration (Ubuntu-compatible
sed)
+ sed -i
"s|^HADOOP_VERSION=.*|HADOOP_VERSION=${params.HADOOP_VERSION}|"
hadoop_install_config.sh
+ sed -i
"s|^HADOOP_HOME=.*|HADOOP_HOME=${HADOOP_HOME}|" hadoop_install_config.sh
+
+ # Comment out chown/sudo chown line before running
install script
+ echo "Commenting out chown command in
install_hadoop.sh..."
+ sed -i 's|^sudo chown|#sudo chown|'
install_hadoop.sh
+
+ ./install_hadoop.sh
+
+ if [ ! -d "${CACHE_DIR}/${HADOOP_DIR}" ]; then
+ echo "Caching extracted Hadoop directory..."
+ cp -r ${HADOOP_HOME} ${CACHE_DIR}/
+ fi
+ . ./hadoop_install_config.sh
+ """
+ }
+ }
+ }
+ }
+ stage('Clone and Build Nutch') {
+ steps {
+ dir("nutch") {
+ git url: 'https://github.com/apache/nutch.git', branch:
'master'
+ sh """
+ set -e
+ REF='${env.RESOLVED_GIT_REF}'
+
+ if echo "\${REF}" | grep -Eq '^pr/[0-9]+\$'; then
+ PR_ID="\${REF#pr/}"
+ echo "Resolving GIT_REF=\${REF} as pull request
\${PR_ID}..."
+ git fetch origin "pull/\${PR_ID}/head:pr-\${PR_ID}"
+ git checkout "pr-\${PR_ID}"
+ echo "Checked out pull request \${PR_ID}"
+ else
+ echo "Fetching tags and heads to resolve
GIT_REF=\${REF}..."
+ git fetch origin --tags
+ git fetch origin
'+refs/heads/*:refs/remotes/origin/*'
+
+ if git rev-parse "refs/tags/\${REF}" >/dev/null
2>&1; then
+ echo "Resolving GIT_REF=\${REF} as tag
(preferred over same-named branch)..."
+ git checkout "refs/tags/\${REF}"
+ echo "Checked out tag \${REF}"
+ elif git rev-parse "refs/remotes/origin/\${REF}"
>/dev/null 2>&1; then
+ echo "Resolving GIT_REF=\${REF} as branch..."
+ git checkout -B "\${REF}" "origin/\${REF}"
+ echo "Checked out branch \${REF}"
+ elif echo "\${REF}" | grep -Eq '^[0-9a-f]{40}\$' \\
+ && git rev-parse --verify "\${REF}^{commit}"
>/dev/null 2>&1; then
+ echo "Resolving GIT_REF=\${REF} as commit..."
+ git checkout "\${REF}"
+ echo "Checked out commit \${REF}"
+ else
+ echo "ERROR: GIT_REF '\${REF}' not found as
tag, branch, or commit on https://github.com/apache/nutch.git";
+ exit 1
+ fi
+ fi
+
+ echo "HEAD=\$(git rev-parse HEAD) describe=\$(git
describe --always --tags)"
+
+ sed -i
'/<name>http.agent.name<\\/name>/,/<\\/property>/d' conf/nutch-site.xml.template
+ sed -i '/<\\/configuration>/i\\
+ <property>\\
+ <name>http.agent.name</name>\\
+ <value>Apache Nutch Single Node Smoke
Test</value>\\
+ </property>' conf/nutch-site.xml.template
+ ant runtime
+ """
+ }
+ }
+ }
+ stage('Start Hadoop Services') {
+ steps {
+ sh '''
+ echo "=== Hadoop configuration ==="
+ cat ${HADOOP_HOME}/etc/hadoop/core-site.xml
+ cat ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml
+ cat ${HADOOP_HOME}/etc/hadoop/yarn-site.xml
+ cat ${HADOOP_HOME}/etc/hadoop/mapred-site.xml
+
+ echo "=== Allow DataNode to register in single-node/CI
(hostname check off) ==="
+ grep -q
'dfs.namenode.datanode.registration.ip-hostname-check'
${HADOOP_HOME}/etc/hadoop/hdfs-site.xml || sed -i '/<\\/configuration>/i\\
+ <property>\\
+ <name>dfs.namenode.datanode.registration.ip-hostname-check</name>\\
+ <value>false</value>\\
+ </property>' ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml
+
+ echo "=== Formatting HDFS namenode ==="
+ hdfs namenode -format -force
+
+ echo "=== Clearing DataNode storage (clusterID must match
freshly formatted NameNode) ==="
+ rm -rf /tmp/hadoop-jenkins/dfs/data
+
+ echo "=== Starting HDFS daemons directly (no SSH) ==="
+ hdfs --daemon start namenode
+ hdfs --daemon start datanode
+
+ echo "=== Starting YARN daemons directly (no SSH) ==="
+ yarn --daemon start resourcemanager
+ yarn --daemon start nodemanager
+
+ echo "=== Waiting for daemons to initialize ==="
+ sleep 5
+
+ echo "=== Waiting for at least one HDFS DataNode to
register ==="
+ HDFS_WAIT_TIMEOUT=90
+ HDFS_WAIT_INTERVAL=3
+ elapsed=0
+ while [ $elapsed -lt $HDFS_WAIT_TIMEOUT ]; do
+ if hdfs dfsadmin -report 2>/dev/null | grep -qE "Live
datanodes \\([1-9][0-9]*\\)"; then
+ echo "HDFS is ready (DataNode registered after
${elapsed}s)."
+ break
+ fi
+ echo "Waiting for DataNode to register... (${elapsed}s /
${HDFS_WAIT_TIMEOUT}s)"
+ sleep $HDFS_WAIT_INTERVAL
+ elapsed=$((elapsed + HDFS_WAIT_INTERVAL))
+ done
+ if [ $elapsed -ge $HDFS_WAIT_TIMEOUT ]; then
+ echo "ERROR: No DataNode registered after
${HDFS_WAIT_TIMEOUT}s. HDFS report:"
+ hdfs dfsadmin -report || true
+ echo "=== DataNode process check ==="
+ jps -l 2>/dev/null | grep -i datanode || ps aux | grep
-i datanode | grep -v grep || true
+ echo "=== Last 200 lines of DataNode log ==="
+ ls -la ${HADOOP_HOME}/logs/*datanode*.log 2>/dev/null ||
true
+ tail -n 200 ${HADOOP_HOME}/logs/*datanode*.log
2>/dev/null || echo "No DataNode log found under ${HADOOP_HOME}/logs/"
+ echo "=== Last 100 lines of NameNode log ==="
+ tail -n 100 ${HADOOP_HOME}/logs/*namenode*.log
2>/dev/null || echo "No NameNode log found under ${HADOOP_HOME}/logs/"
+ exit 1
+ fi
+
+ echo "=== Verifying Hadoop daemons ==="
+ hdfs dfsadmin -report || echo "WARNING: HDFS report failed"
+ yarn node -list || echo "WARNING: YARN node list failed"
+ '''
+ }
+ }
+ stage('Smoke Test Nutch Core Tools') {
+ steps {
+ sh """
+ sed -i '/nutch index crawl\\/crawldb -linkdb
crawl\\/linkdb -dir crawl\\/segments/s/^/#/' test_nutch_tools.sh
+ sed -i '/nutch clean crawl\\/crawldb/s/^/#/'
test_nutch_tools.sh
+ ./test_nutch_tools.sh
+ """
+ }
+ }
+ stage('Stop Hadoop Services') {
+ steps {
+ sh '''
+ echo "=== Stopping YARN daemons ==="
+ yarn --daemon stop nodemanager || true
+ yarn --daemon stop resourcemanager || true
+
+ echo "=== Stopping HDFS daemons ==="
+ hdfs --daemon stop datanode || true
+ hdfs --daemon stop namenode || true
+ '''
+ }
+ }
+ }
+ post {
+ always {
+ script {
+ if (params.EMAIL_RECIPIENT != '') {
+ emailext (
+ to: params.EMAIL_RECIPIENT,
+ subject: "Jenkins Job ${env.JOB_NAME}
#${env.BUILD_NUMBER} - ${currentBuild.currentResult}",
+ body: """
+ Build ${currentBuild.currentResult}
+ Job: ${env.JOB_NAME}
+ Build Number: ${env.BUILD_NUMBER}
+ Git Ref: ${env.RESOLVED_GIT_REF ?: params.GIT_REF
?: 'master'}
+ Hadoop Version: ${params.HADOOP_VERSION}
+ Build URL: ${env.BUILD_URL}
+ """
+ )
+ }
+ }
+ }
+ }
+}
