This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch HBASE-30004 in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 0ebfa0a206cef0a77cbbea4038ed0d7f35b309d9 Author: Duo Zhang <[email protected]> AuthorDate: Tue Mar 17 19:08:44 2026 +0800 HBASE-30004 Implement a new generate web site jenkins job --- dev-support/docker/Dockerfile | 1 + .../generate-website/build-hbase-website.sh | 250 +++++++++++++++++++++ .../generate-hbase-website.Jenkinsfile | 134 +++++++++++ 3 files changed, 385 insertions(+) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index d5d33987485..294fc272f1c 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -57,6 +57,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -qq update && \ shellcheck='0.8.0-*' \ libxml2-dev='2.9.13+dfsg-*' \ libxml2-utils='2.9.13+dfsg-*' \ + zip='3.0-*' \ && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ diff --git a/dev-support/generate-website/build-hbase-website.sh b/dev-support/generate-website/build-hbase-website.sh new file mode 100755 index 00000000000..4e28ff757de --- /dev/null +++ b/dev-support/generate-website/build-hbase-website.sh @@ -0,0 +1,250 @@ +#!/bin/bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# This script is meant to run as part of a Jenkins job such as +# https://builds.apache.org/job/hbase_generate_website/ + +set -e +function usage { + echo "Usage: ${0} [options] /path/to/hbase/checkout" + echo "" + echo " --working-dir /path/to/use Path for writing logs and a local checkout of hbase-site repo." + echo " if given must exist." + echo " defaults to making a directory via mktemp." + echo " --local-repo /path/for/maven/.m2 Path for putting local maven repo." + echo " if given must exist." + echo " defaults to making a clean directory in --working-dir." + echo " --help show this usage message." + exit 1 +} +# if no args specified, show usage +if [ $# -lt 1 ]; then + usage +fi + +# Get arguments +declare component_dir +declare working_dir +declare local_repo +while [ $# -gt 0 ] +do + case "$1" in + --working-dir) shift; working_dir=$1; shift;; + --local-repo) shift; local_repo=$1; shift;; + --) shift; break;; + -*) usage ;; + *) break;; # terminate while loop + esac +done + +# should still have where component checkout is. +if [ $# -lt 1 ]; then + usage +fi + +MVN="mvn" +if ! command -v mvn &>/dev/null; then + MVN=$MAVEN_HOME/bin/mvn +fi + +component_dir="$(cd "$(dirname "$1")"; pwd)/$(basename "$1")" + +if [ -z "${working_dir}" ]; then + echo "[DEBUG] defaulting to creating a directory via mktemp" + if ! working_dir="$(mktemp -d -t hbase-generate-website)" ; then + echo "Failed to create temporary working directory. Please specify via --working-dir" + exit 1 + fi +else + # absolutes please + working_dir="$(cd "$(dirname "${working_dir}")"; pwd)/$(basename "${working_dir}")" + if [ ! -d "${working_dir}" ]; then + echo "passed working directory '${working_dir}' must already exist." + exit 1 + fi +fi + +echo "You'll find logs and temp files in ${working_dir}" + +if [ -z "${local_repo}" ]; then + echo "[DEBUG] defaulting to creating a local repo within '${working_dir}'" + local_repo="${working_dir}/.m2/repo" + # Nuke the local maven repo each time, to start with a known environment + rm -Rf "${local_repo}" + mkdir -p "${local_repo}" +else + # absolutes please + local_repo="$(cd "$(dirname "${local_repo}")"; pwd)/$(basename "${local_repo}")" + if [ ! -d "${local_repo}" ]; then + echo "passed directory for storing the maven repo '${local_repo}' must already exist." + exit 1 + fi +fi + +export MAVEN_OPTS="${MAVEN_OPTS} -Dmaven.repo.local=${local_repo}" + +# Verify the Maven version +${MVN} -version +# Verify the git version +git --version + +cd "${working_dir}" + +# Clean any leftover files in case we are reusing the workspace +rm -Rf -- *.patch *.patch.zip target *.txt hbase-site + +# Save and print the SHA we are building +CURRENT_HBASE_COMMIT="$(cd "${component_dir}" && git rev-parse HEAD)" +# Fail if it's empty +if [ -z "${CURRENT_HBASE_COMMIT}" ]; then + echo "Got back a blank answer for the current HEAD. failing." + exit 1 +fi +echo "Current HBase commit: $CURRENT_HBASE_COMMIT" + +# Clone the hbase-site repo manually so it doesn't trigger spurious +# commits in Jenkins. +git clone --depth 1 --branch asf-site https://gitbox.apache.org/repos/asf/hbase-site.git + +# Figure out if the commit of the hbase repo has already been built and bail if so. +declare -i PUSHED +PUSHED=$(cd hbase-site && git rev-list --grep "${CURRENT_HBASE_COMMIT}" --fixed-strings --count HEAD) +echo "[DEBUG] hash was found in $PUSHED commits for hbase-site repository." + +if [ "${PUSHED}" -ne 0 ]; then + echo "$CURRENT_HBASE_COMMIT is already mentioned in the hbase-site commit log. Not building." + exit 0 +else + echo "$CURRENT_HBASE_COMMIT is not yet mentioned in the hbase-site commit log. Assuming we don't have it yet." +fi + +# Go to the hbase directory so we can build the site +cd "${component_dir}" + +# This will only be set for builds that are triggered by SCM change, not manual builds +if [ -n "$CHANGE_ID" ]; then + echo -n " ($CHANGE_ID - $CHANGE_TITLE)" +fi + +# Build and install HBase, then build the site +echo "Building HBase" +# TODO we have to do a local install first because for whatever reason, the maven-javadoc-plugin's +# forked compile phase requires that test-scoped dependencies be available, which +# doesn't work since we will not have done a test-compile phase (MJAVADOC-490). the first place this +# breaks for me is hbase-server trying to find hbase-http:test and hbase-zookeeper:test. +# But! some sunshine: because we're doing a full install before running site, we can skip all the +# compiling in the forked executions. We have to do it awkwardly because MJAVADOC-444. +if ${MVN} \ + --batch-mode \ + -Psite-install-step \ + --errors \ + --log-file="${working_dir}/hbase-install-log-${CURRENT_HBASE_COMMIT}.txt" \ + clean install \ + && ${MVN} site \ + --batch-mode \ + -Dscala.skip=true \ + -Psite-build-step \ + --errors \ + --log-file="${working_dir}/hbase-site-log-${CURRENT_HBASE_COMMIT}.txt"; then + echo "Successfully built site." +else + status=$? + echo "Maven commands to build the site failed. check logs for details ${working_dir}/hbase-*-log-*.txt" + exit $status +fi + +# Stage the site +echo "Staging HBase site" +${MVN} \ + --batch-mode \ + --errors \ + --log-file="${working_dir}/hbase-stage-log-${CURRENT_HBASE_COMMIT}.txt" \ + site:stage +status=$? +if [ $status -ne 0 ] || [ ! -d target/staging ]; then + echo "Failure: mvn site:stage" + exit $status +fi + +# Get ready to update the hbase-site repo with the new artifacts +cd "${working_dir}/hbase-site" + +#Remove previously-generated files +FILES_TO_REMOVE=("hbase-*" + "apidocs" + "devapidocs" + "testapidocs" + "testdevapidocs" + "xref" + "xref-test" + "*book*" + "*.html" + "*.pdf*" + "css" + "js" + "images") + +for FILE in "${FILES_TO_REMOVE[@]}"; do + if [ -e "${FILE}" ]; then + echo "Removing hbase-site/$FILE" + rm -Rf "${FILE}" + fi +done + +# Copy in the newly-built artifacts +# First copy documentation from Maven site build +echo "Copying documentation from target/staging" +# TODO what do we do when the site build wants to remove something? Can't rsync because e.g. release-specific docs. +cp -pPR "${component_dir}"/target/staging/* . + +# Then copy the new website (landing page) from hbase-website/build/client +echo "Copying new website from hbase-website/build/client" +cp -pPR "${component_dir}"/hbase-website/build/client/* . + +# If the index.html is missing, bail because this is serious +if [ ! -f index.html ]; then + echo "The index.html is missing. Aborting." + exit 1 +fi + +echo "Adding all the files we know about" +git add . +if [[ -z "$(git status --porcelain)" ]]; then + echo "No files to commit, skipping..." + exit 0 +fi +# Create the commit message and commit the changes +WEBSITE_COMMIT_MSG="Published site at $CURRENT_HBASE_COMMIT." +echo "WEBSITE_COMMIT_MSG: $WEBSITE_COMMIT_MSG" +git commit -m "${WEBSITE_COMMIT_MSG}" -a +# Dump a little report +echo "This commit changed these files (excluding Modified files):" +git diff --name-status --diff-filter=ADCRTXUB origin/asf-site | tee "${working_dir}/hbase-file-diff-summary-${CURRENT_HBASE_COMMIT}.txt" +# Create a patch, which Jenkins can save as an artifact and can be examined for debugging +git format-patch --stdout origin/asf-site > "${working_dir}/${CURRENT_HBASE_COMMIT}.patch" +if [ ! -s "${working_dir}/${CURRENT_HBASE_COMMIT}.patch" ]; then + echo "Something went wrong when creating the patch of our updated site." + exit 1 +fi +echo "Change set saved to patch ${working_dir}/${CURRENT_HBASE_COMMIT}.patch" + +# Zip up the patch so Jenkins can save it +cd "${working_dir}" +zip website.patch.zip "${CURRENT_HBASE_COMMIT}.patch" diff --git a/dev-support/generate-website/generate-hbase-website.Jenkinsfile b/dev-support/generate-website/generate-hbase-website.Jenkinsfile new file mode 100644 index 00000000000..3155fba6d32 --- /dev/null +++ b/dev-support/generate-website/generate-hbase-website.Jenkinsfile @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +pipeline { + agent { + node { + label 'hbase' + } + } + triggers { + pollSCM('@daily') + } + options { + buildDiscarder(logRotator(numToKeepStr: '30')) + timeout (time: 1, unit: 'HOURS') + timestamps() + skipDefaultCheckout() + disableConcurrentBuilds() + } + parameters { + booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a lot more meta-information.') + booleanParam(name: 'FORCE_FAIL', defaultValue: false, description: 'force a failure to test notifications.') + } + stages { + stage ('build hbase website') { + steps { + dir('component') { + checkout scm + } + sh '''#!/bin/bash -e + if [ "${DEBUG}" = "true" ]; then + set -x + fi + if [ "${FORCE_FAIL}" = "true" ]; then + false + fi + user=$(whoami) + docker build -t hbase-build-website -f "${WORKSPACE}/component/dev-support/docker/Dockerfile" . + docker run --rm -v "${WORKSPACE}":/home/${user} -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro \ + -u `id -u`:`id -g` -e JAVA_HOME="/usr/lib/jvm/java-17" -e GIT_AUTHOR_NAME="HBase" \ + -e GIT_AUTHOR_EMAIL="[email protected]" -e GIT_COMMITTER_NAME="HBase" \ + -e GIT_COMMITTER_EMAIL="[email protected]" --workdir=/home/${user} hbase-build-website \ + "component/dev-support/generate-website/build-hbase-website.sh" \ + --working-dir /home/${user} component + ''' + script { + if (fileExists('website.patch.zip')) { + sh'''#!/bin/bash -e + patch=$(ls -1 *.patch | head -n 1) + echo "Has patch ${patch}, stash and then publish" + ''' + stash name: 'patch', includes: "*.patch" + env.PUBLISH_WEBSITE = "true" + } else { + echo "No patch file, skip stashing and publishing" + env.PUBLISH_WEBSITE = "false" + } + } + } + } + stage('publish hbase website') { + agent { + node { + label 'git-websites' + } + } + when { + expression { + return env.PUBLISH_WEBSITE == 'true' + } + } + steps { + unstash 'patch' + sh '''#!/bin/bash -e + # wipe out stall repo + rm -rf hbase-site + git clone --depth 1 --branch asf-site https://gitbox.apache.org/repos/asf/hbase-site.git + patch=$(ls -1 *.patch | head -n 1) + cd hbase-site; + echo "applying ${patch}" + git am ../${patch} + echo "Publishing changes to remote repo..." + if git push origin asf-site; then + echo "changes pushed." + else + echo "Failed to push to asf-site. Website not updated." + exit 1 + fi + echo "Sending empty commit to work around INFRA-10751." + git commit --allow-empty -m "INFRA-10751 Empty commit" + # Push the empty commit + if git push origin asf-site; then + echo "empty commit pushed." + else + echo "Failed to push the empty commit to asf-site. Website may not update. Manually push an empty commit to fix this. (See INFRA-10751)" + exit 1 + fi + echo "Pushed the changes to branch asf-site. Refresh http://hbase.apache.org/ to see the changes within a few minutes." + ''' + } + } + } + post { + always { + // Has to be relative to WORKSPACE. + archiveArtifacts artifacts: '*.patch.zip,hbase-*.txt' + } + failure { + mail to: '[email protected]', replyTo: '[email protected]', subject: "Failure: HBase Generate Website", body: """ +Build status: ${currentBuild.currentResult} + +The HBase website has not been updated to incorporate recent HBase changes. + +See ${env.BUILD_URL}console +""" + } + cleanup { + deleteDir() + } + } +}
