commit: b291ea47548bf025259632a18d86e16d9bb0a886
Author: Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Tue Jan 16 01:11:44 2018 +0000
Commit: Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Tue Jan 16 01:11:44 2018 +0000
URL: https://gitweb.gentoo.org/proj/releng.git/commit/?id=b291ea47
catalyst-auto: move all code into functions
Mixing inline funcs and executable code makes it hard to follow and
shuffle ordering of operations. Put everything other than variables
into functions.
The delta here is large, but it's almost entirely indentation changes.
This also makes updating the script (like `git pull`) more resilient.
tools/catalyst-auto | 536 ++++++++++++++++++++++++++--------------------------
1 file changed, 273 insertions(+), 263 deletions(-)
diff --git a/tools/catalyst-auto b/tools/catalyst-auto
index 02115d1c..cf5a7c65 100755
--- a/tools/catalyst-auto
+++ b/tools/catalyst-auto
@@ -19,6 +19,17 @@ unset UNSHARE
CATALYST_CONFIG=/etc/catalyst/catalyst.conf
+# Probe the default source dir from this script name.
+REPO_DIR=$(dirname "$(dirname "$(realpath "$0")")")
+
+# Set up defaults that config files can override if they want.
+SUBARCH=$(uname -m)
+EMAIL_TO="rel...@gentoo.org,gentoo-releng-autobui...@lists.gentoo.org"
+# Use full hostname by default as Gentoo servers will reject short names.
+EMAIL_FROM="catalyst@$(hostname -f)"
+EMAIL_SUBJECT_PREPEND="[${SUBARCH}-auto]"
+
+# Variables updated by command line arguments.
declare -a config_files
config_files=()
verbose=0
@@ -28,9 +39,6 @@ preclean=0
lastrun=0
lock_file=
-# Set pipefail so that run_cmd returns the right value in $?
-set -o pipefail
-
usage() {
local msg=$1
@@ -125,269 +133,271 @@ trigger_post_build() {
fi
}
-# Parse args
-while [ ${#} -gt 0 ]
-do
- a=${1}
- shift
- case "${a}" in
- -h|--help)
- usage
- exit 0
- ;;
- -c|--config)
- config_files+=("$1")
- shift
- ;;
- -v|--verbose)
- verbose=$(($verbose+1))
- ;;
- -k|--keep-tmpdir)
- keep_tmpdir=1
- ;;
- -t|--test)
- testing=1
- ;;
- -C|--preclean)
- preclean=1
- ;;
- --interval)
- lastrun=$1
- shift
- ;;
- -l|--lock)
- lock_file=$1
- shift
- ;;
- -*)
- usage "ERROR: You have specified an invalid option: ${a}"
- exit 1
- ;;
- *)
- usage "ERROR: This script takes no arguments: ${a}"
- exit 1
- ;;
- esac
-done
-
-(
-
-if [[ -n ${lock_file} ]]; then
- if ! flock -n 9; then
- echo "catalyst-auto already running"
- exit 1
- fi
-fi
-
-# Probe the default source dir from this script name.
-REPO_DIR=$(dirname "$(dirname "$(realpath "$0")")")
-
-# Set up defaults that config files can override if they want.
-SUBARCH=$(uname -m)
-EMAIL_TO="rel...@gentoo.org,gentoo-releng-autobui...@lists.gentoo.org"
-# Use full hostname by default as Gentoo servers will reject short names.
-EMAIL_FROM="catalyst@$(hostname -f)"
-EMAIL_SUBJECT_PREPEND="[${SUBARCH}-auto]"
-
-doneconfig=0
-for config_file in "${config_files[@]}"; do
- # Make sure all required values were specified
- if [ -z "${config_file}" -o ! -e "${config_file}" ]; then
- usage "ERROR: You must specify a valid config file to use:
'$config_file' is not valid"
- exit 1
- fi
- source "${config_file}"
- doneconfig=1
-done
-if [[ $doneconfig -eq 0 ]]; then
- usage "ERROR: You must specify at least one valid config file to use"
- exit 1
-fi
-
-# Some configs will set this explicitly, so don't clobber it.
-: ${BUILD_SRCDIR_BASE:=$(catalyst_var storedir)}
-
-# See if we had a recent success.
-if [[ ${lastrun} -ne 0 ]]; then
- last_success_file="${BUILD_SRCDIR_BASE}/.last_success"
- delay=$(( lastrun * 24 * 60 * 60 ))
- last_success=$(head -1 "${last_success_file}" 2>/dev/null || echo 0)
- if [[ $(date +%s) -lt $(( last_success + delay )) ]]; then
- exit 0
- fi
-fi
-
-DATESTAMP=$(date -u +%Y%m%d)
-TIMESTAMP=$(date -u +%Y%m%dT%H%M%SZ)
-TMPDIR=$(mktemp -d --tmpdir="${TMP_PATH:-/tmp}"
"catalyst-auto.${TIMESTAMP}.XXXXXX")
-
-# Nuke any previous tmpdirs to keep them from accumulating.
-if [[ ${preclean} -eq 1 ]]; then
- rm -rf "${TMPDIR%.??????}".*
- mkdir "${TMPDIR}"
-fi
-
-if [ ${verbose} -ge 1 ]; then
- echo "TMPDIR = ${TMPDIR}"
- echo "DATESTAMP = ${DATESTAMP}"
- echo "TIMESTAMP = ${TIMESTAMP}"
-fi
-
-if ! mkdir -p "${TMPDIR}"/{specs,kconfig,log}; then
- echo "Couldn't create tempdirs!"
- exit 1
-fi
-
-if ! run_cmd "${TMPDIR}/log/pre_build.log" pre_build; then
- send_email "Catalyst build error - pre_build" "The pre_build function
failed" "${TMPDIR}/log/pre_build.log"
- exit 1
-fi
-
-cd "${SPECS_DIR}" || exit 1
-
-for a in "" ${SETS}; do
- if [ -z "${a}" ]; then
- specs_var="SPECS"
- optional_specs_var="OPTIONAL_SPECS"
- else
- specs_var="SET_${a}_SPECS"
- optional_specs_var="SET_${a}_OPTIONAL_SPECS"
- fi
-
- for i in ${!specs_var} ${!optional_specs_var}; do
- cp --parents "${i}" "${TMPDIR}"/specs/
- done
-done
-
-find "${KCONFIG_DIR}" -type f -exec cp {} "${TMPDIR}"/kconfig \;
-
-cd "${TMPDIR}/specs" || exit 1
-
-# Fix up specs with datestamp
-for i in $(find -name '*.spec'); do
- # Grab current version_stamp and source_subpath
- old_version_stamp=$(grep version_stamp "${i}" | sed -e 's|^version_stamp:
*||')
- old_source_subpath=$(grep source_subpath "${i}" | sed -e 's|^source_subpath:
*||')
-
- new_version_stamp=$(echo "${old_version_stamp}" | sed -e
"s|^\(.*-\)\?.*$|\1${TIMESTAMP}|")
- new_source_subpath=$(echo "${old_source_subpath}" | sed -e
"s|${old_version_stamp}|${new_version_stamp}|")
-
- sed -i "s|^version_stamp:.*$|version_stamp: ${new_version_stamp}|" "${i}"
- sed -i "s|^snapshot:.*$|snapshot: ${TIMESTAMP}|" "${i}"
-
- # We don't want to mangle the source_subpath for our stage1 spec
- if ! grep -q '^target: *stage[14]$' "${i}"; then
- sed -i "s|^source_subpath:.*$|source_subpath: ${new_source_subpath}|"
"${i}"
- fi
-
- sed -i "/^livecd\/iso/s|${old_version_stamp}|${new_version_stamp}|" "${i}"
- sed -i "/^livecd\/volid/s|${old_version_stamp}|${new_version_stamp}|" "${i}"
-
- kconfig_lines=$(grep '^boot/kernel/[^/]\+/config:' "${i}")
- if [ -n "${kconfig_lines}" ]; then
- echo "${kconfig_lines}" | while read line; do
- key=$(echo "${line}" | cut -d: -f1)
- filename=$(basename $(echo "${line}" | cut -d: -f2))
- sed -i "s|^${key}:.*\$|${key}: ${TMPDIR}/kconfig/${filename}|" "${i}"
- done
- fi
-
- # Expand vars that the spec expects us to.
- sed -i \
- -e "s:@DATESTAMP@:${DATESTAMP}:g" \
- -e "s:@TIMESTAMP@:${TIMESTAMP}:g" \
- -e "s:@REPO_DIR@:${REPO_DIR}:g" \
- "${i}"
-done
-
-if [ "${testing}" -eq 1 ]; then
- echo "Exiting due to --test"
- exit
-fi
-
-if [[ ${preclean} -eq 1 ]]; then
- snapshot_cache=$(catalyst_var snapshot_cache)
- if [[ -z ${snapshot_cache} ]]; then
- echo "error: snapshot_cache not set in config file"
- exit 1
- fi
- pushd "${BUILD_SRCDIR_BASE}" >/dev/null || exit 1
- rm -rf --one-file-system \
- kerncache packages snapshots tmp "${snapshot_cache}"/*
- popd >/dev/null
-fi
-
-# Create snapshot
-if ! run_cmd "${TMPDIR}/log/snapshot.log" catalyst -c "${CATALYST_CONFIG}" -s
"${TIMESTAMP}"; then
- send_email "Catalyst build error - snapshot" "" "${TMPDIR}/log/snapshot.log"
- exit 1
-fi
-
-build_failure=0
+parse_args() {
+ local a
+ while [[ $# -gt 0 ]] ; do
+ a=$1
+ shift
+ case "${a}" in
+ -h|--help)
+ usage
+ exit 0
+ ;;
+ -c|--config)
+ config_files+=("$1")
+ shift
+ ;;
+ -v|--verbose)
+ verbose=$(($verbose+1))
+ ;;
+ -k|--keep-tmpdir)
+ keep_tmpdir=1
+ ;;
+ -t|--test)
+ testing=1
+ ;;
+ -C|--preclean)
+ preclean=1
+ ;;
+ --interval)
+ lastrun=$1
+ shift
+ ;;
+ -l|--lock)
+ lock_file=$1
+ shift
+ ;;
+ -*)
+ usage "ERROR: You have specified an invalid option:
${a}"
+ exit 1
+ ;;
+ *)
+ usage "ERROR: This script takes no arguments: ${a}"
+ exit 1
+ ;;
+ esac
+ done
+}
-timeprefix=()
-which time >/dev/null && timeprefix=( "time" )
+run_catalyst_commands() {
+ doneconfig=0
+ for config_file in "${config_files[@]}"; do
+ # Make sure all required values were specified.
+ if [[ -z "${config_file}" || ! -e "${config_file}" ]]; then
+ usage "ERROR: You must specify a valid config file to
use: '$config_file' is not valid"
+ exit 1
+ fi
+ source "${config_file}"
+ doneconfig=1
+ done
+ if [[ ${doneconfig} == 0 ]]; then
+ usage "ERROR: You must specify at least one valid config file
to use"
+ exit 1
+ fi
+
+ # Some configs will set this explicitly, so don't clobber it.
+ : ${BUILD_SRCDIR_BASE:=$(catalyst_var storedir)}
+
+ # See if we had a recent success.
+ if [[ ${lastrun} != 0 ]]; then
+ last_success_file="${BUILD_SRCDIR_BASE}/.last_success"
+ delay=$(( lastrun * 24 * 60 * 60 ))
+ last_success=$(head -1 "${last_success_file}" 2>/dev/null ||
echo 0)
+ if [[ $(date +%s) -lt $(( last_success + delay )) ]]; then
+ exit 0
+ fi
+ fi
+
+ DATESTAMP=$(date -u +%Y%m%d)
+ TIMESTAMP=$(date -u +%Y%m%dT%H%M%SZ)
+ TMPDIR=$(mktemp -d --tmpdir="${TMP_PATH:-/tmp}"
"catalyst-auto.${TIMESTAMP}.XXXXXX")
+
+ # Nuke any previous tmpdirs to keep them from accumulating.
+ if [[ ${preclean} == 1 ]]; then
+ rm -rf "${TMPDIR%.??????}".*
+ mkdir "${TMPDIR}"
+ fi
+
+ if [[ ${verbose} -ge 1 ]]; then
+ echo "TMPDIR = ${TMPDIR}"
+ echo "DATESTAMP = ${DATESTAMP}"
+ echo "TIMESTAMP = ${TIMESTAMP}"
+ fi
+
+ if ! mkdir -p "${TMPDIR}"/{specs,kconfig,log}; then
+ echo "Couldn't create tempdirs!"
+ exit 1
+ fi
+
+ if ! run_cmd "${TMPDIR}/log/pre_build.log" pre_build; then
+ send_email "Catalyst build error - pre_build" "The pre_build
function failed" "${TMPDIR}/log/pre_build.log"
+ exit 1
+ fi
+
+ cd "${SPECS_DIR}" || exit 1
+
+ for a in "" ${SETS}; do
+ if [[ -z "${a}" ]]; then
+ specs_var="SPECS"
+ optional_specs_var="OPTIONAL_SPECS"
+ else
+ specs_var="SET_${a}_SPECS"
+ optional_specs_var="SET_${a}_OPTIONAL_SPECS"
+ fi
+
+ for i in ${!specs_var} ${!optional_specs_var}; do
+ cp --parents "${i}" "${TMPDIR}"/specs/
+ done
+ done
+
+ find "${KCONFIG_DIR}" -type f -exec cp {} "${TMPDIR}"/kconfig \;
+
+ cd "${TMPDIR}/specs" || exit 1
+
+ # Fix up specs with datestamp
+ for i in $(find -name '*.spec'); do
+ # Grab current version_stamp and source_subpath
+ old_version_stamp=$(grep version_stamp "${i}" | sed -e
's|^version_stamp: *||')
+ old_source_subpath=$(grep source_subpath "${i}" | sed -e
's|^source_subpath: *||')
+
+ new_version_stamp=$(echo "${old_version_stamp}" | sed -e
"s|^\(.*-\)\?.*$|\1${TIMESTAMP}|")
+ new_source_subpath=$(echo "${old_source_subpath}" | sed -e
"s|${old_version_stamp}|${new_version_stamp}|")
+
+ sed -i "s|^version_stamp:.*$|version_stamp:
${new_version_stamp}|" "${i}"
+ sed -i "s|^snapshot:.*$|snapshot: ${TIMESTAMP}|" "${i}"
+
+ # We don't want to mangle the source_subpath for our stage1 spec
+ if ! grep -q '^target: *stage[14]$' "${i}"; then
+ sed -i "s|^source_subpath:.*$|source_subpath:
${new_source_subpath}|" "${i}"
+ fi
+
+ sed -i
"/^livecd\/iso/s|${old_version_stamp}|${new_version_stamp}|" "${i}"
+ sed -i
"/^livecd\/volid/s|${old_version_stamp}|${new_version_stamp}|" "${i}"
+
+ kconfig_lines=$(grep '^boot/kernel/[^/]\+/config:' "${i}")
+ if [[ -n ${kconfig_lines} ]]; then
+ echo "${kconfig_lines}" | while read line; do
+ key=$(echo "${line}" | cut -d: -f1)
+ filename=$(basename $(echo "${line}" | cut -d:
-f2))
+ sed -i "s|^${key}:.*\$|${key}:
${TMPDIR}/kconfig/${filename}|" "${i}"
+ done
+ fi
+
+ # Expand vars that the spec expects us to.
+ sed -i \
+ -e "s:@DATESTAMP@:${DATESTAMP}:g" \
+ -e "s:@TIMESTAMP@:${TIMESTAMP}:g" \
+ -e "s:@REPO_DIR@:${REPO_DIR}:g" \
+ "${i}"
+ done
+
+ if [[ ${testing} == 1 ]]; then
+ echo "Exiting due to --test"
+ exit
+ fi
+
+ if [[ ${preclean} == 1 ]]; then
+ snapshot_cache=$(catalyst_var snapshot_cache)
+ if [[ -z ${snapshot_cache} ]]; then
+ echo "error: snapshot_cache not set in config file"
+ exit 1
+ fi
+ pushd "${BUILD_SRCDIR_BASE}" >/dev/null || exit 1
+ rm -rf --one-file-system \
+ kerncache packages snapshots tmp "${snapshot_cache}"/*
+ popd >/dev/null
+ fi
+
+ # Create snapshot
+ if ! run_cmd "${TMPDIR}/log/snapshot.log" catalyst -c
"${CATALYST_CONFIG}" -s "${TIMESTAMP}"; then
+ send_email "Catalyst build error - snapshot" ""
"${TMPDIR}/log/snapshot.log"
+ exit 1
+ fi
+
+ build_failure=0
+
+ timeprefix=()
+ which time >/dev/null && timeprefix=( "time" )
+
+ for a in "" ${SETS}; do
+ if [[ -z ${a} ]]; then
+ specs_var="SPECS"
+ optional_specs_var="OPTIONAL_SPECS"
+ else
+ specs_var="SET_${a}_SPECS"
+ optional_specs_var="SET_${a}_OPTIONAL_SPECS"
+ fi
+
+ for i in ${!specs_var}; do
+ LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:'
-e 's:\.spec$::').log"
+ run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -c
"${CATALYST_CONFIG}" -f "${i}"
+ if [[ $? != 0 ]]; then
+ build_failure=1
+ send_email "Catalyst fatal build error - ${i}"
"" "${LOGFILE}"
+ continue 2
+ else
+ trigger_post_build
+ fi
+ done
+
+ for i in ${!optional_specs_var}; do
+ LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:'
-e 's:\.spec$::').log"
+ run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -c
"${CATALYST_CONFIG}" -f "${i}"
+ if [[ $? != 0 ]]; then
+ build_failure=1
+ send_email "Catalyst non-fatal build error -
${i}" "" "${LOGFILE}"
+ break
+ else
+ trigger_post_build
+ fi
+ done
+
+ for i in ${!specs_var} ${!optional_specs_var}; do
+ LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:'
-e 's:\.spec$::')_purge.log"
+ run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst
--purgetmponly -c "${CATALYST_CONFIG}" -f "${i}"
+ done
+
+ update_symlinks
+ done
+
+ trigger_post_build
+
+ if [[ ${build_failure} == 0 ]]; then
+ if [[ ${lastrun} != 0 ]]; then
+ stamp=$(date)
+ (date -d"${stamp}" +%s; echo "${stamp}")
>"${last_success_file}"
+ fi
+
+ send_email "Catalyst build success" "Build process complete."
+
+ if [[ ${keep_tmpdir} == 0 ]]; then
+ if ! rm -rf "${TMPDIR}"; then
+ echo "Could not remove tmpdir ${TMPDIR}!"
+ exit 1
+ fi
+ fi
+ else
+ send_email "Catalyst build complete, but with errors" "Build
process has completed, but there were errors. Please consult previous emails to
determine the problem."
+ fi
+}
-for a in "" ${SETS}; do
- if [ -z "${a}" ]; then
- specs_var="SPECS"
- optional_specs_var="OPTIONAL_SPECS"
- else
- specs_var="SET_${a}_SPECS"
- optional_specs_var="SET_${a}_OPTIONAL_SPECS"
- fi
+main() {
+ # Set pipefail so that run_cmd returns the right value in $?.
+ set -o pipefail
- for i in ${!specs_var}; do
- LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e
's:\.spec$::').log"
- run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -c
"${CATALYST_CONFIG}" -f "${i}"
- if [ $? != 0 ]; then
- build_failure=1
- send_email "Catalyst fatal build error - ${i}" "" "${LOGFILE}"
- continue 2
- else
- trigger_post_build
- fi
- done
-
- for i in ${!optional_specs_var}; do
- LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e
's:\.spec$::').log"
- run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -c
"${CATALYST_CONFIG}" -f "${i}"
- if [ $? != 0 ]; then
- build_failure=1
- send_email "Catalyst non-fatal build error - ${i}" "" "${LOGFILE}"
- break
- else
- trigger_post_build
- fi
- done
-
- for i in ${!specs_var} ${!optional_specs_var}; do
- LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e
's:\.spec$::')_purge.log"
- run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst --purgetmponly -c
"${CATALYST_CONFIG}" -f "${i}"
- done
-
- update_symlinks
-done
-
-trigger_post_build
-
-if [ ${build_failure} = 0 ]; then
- if [[ ${lastrun} -ne 0 ]]; then
- stamp=$(date)
- (date -d"${stamp}" +%s; echo "${stamp}") >"${last_success_file}"
- fi
+ # Parse user arguments before we try doing container logic.
+ parse_args "$@"
- send_email "Catalyst build success" "Build process complete."
+ (
+ if [[ -n ${lock_file} ]]; then
+ if ! flock -n 9; then
+ echo "catalyst-auto already running"
+ exit 1
+ fi
+ fi
- if [ "${keep_tmpdir}" = 0 ]; then
- if ! rm -rf "${TMPDIR}"; then
- echo "Could not remove tmpdir ${TMPDIR}!"
- exit 1
- fi
- fi
-
-else
- send_email "Catalyst build complete, but with errors" "Build process has
completed, but there were errors. Please consult previous emails to determine
the problem."
-fi
+ run_catalyst_commands
+ ) 9>"${lock_file:-/dev/null}"
+}
-) 9>"${lock_file:-/dev/null}"
+main "$@"
