Hi, here's a patch against mp-buildbot that would add a fail cache for use on the buildbots. Unfortunately I don't have a test setup yet, so I'm hesitating to commit this immediately. I'd welcome feedback, a code review, or testing by somebody with a working builbot setup.
Basically, I'm keeping failcache per tuple of (portname, canonical variants, checksum) where - I need one additional call to a custom port client to dump the canonical variants - calculate the checksum as sha256(sort(sha256(Portfile), sha256(files/*))) which will make the buildbots re-try a dependency if, for example, a maintainer forgot to commit a patch file Index: functions =================================================================== --- functions (nonexistent) +++ functions (working copy) @@ -0,0 +1,128 @@ +#!/bin/bash +# -*- coding: utf-8; mode: sh; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:ft=sh:et:sw=4:ts=4:sts=4 + +# Helper functions for mp-buildbot + +# Print $0 and arguments to standard error. +# Unset IFS to ensure "$*" uses spaces as separators. +msg() (unset IFS; printf >&2 '%s: %s\n' "$0" "$*") +err() { msg error: "$@"; } +warn() { msg warning: "$@"; } + +## Compute a failcache hash for the given port +# +# Computes and prints a hash uniquely identifying a specific state of a port's +# definition files, including the Portfile's hash as well as the port's +# patchfiles. To build the hash, this function executes the following +# algorithm: +# - For the Portfile, and each file in files/ (if any), calculate a SHA256 +# hash +# - Sort the hash values alphabetically +# - Hash the result using SHA256 +# This means a failcache entry will not match if a patchfile changes. A common +# case where this is the desired behavior is a port committed without +# a required patchfile. +# +# Valid arguments are all arguments accepted by "port dir". +compute_failcache_hash() { + local portdir + local -a filelist + + portdir=$("${option_prefix}/bin/port" dir "$@") + if [ $? -ne 0 ] || [ -z "$portdir" ]; then + err "Could not compute failcache hash: port dir" "$@" "failed" + return 1 + fi + + if [ ! -d "$portdir" ]; then + err "Port directory $portdir does not exist" + return 2 + fi + + filelist=("$portdir/Portfile") + if [ -d "$portdir/files" ]; then + filelist+=("$portdir/files") + fi + + find "${filelist[@]}" -type f -exec openssl dgst -sha256 {} \; |\ + cut -d' ' -f2 |\ + sort |\ + openssl dgst -sha256 |\ + cut -d' ' -f2 +} + +## Compute a key that uniquely identifies a (port, variants, portfile-hash) tuple +# +# Valid arguments are a port name, optionally followed by a variant +# specification. Invokes "port dir" to find the Portfile and patchfiles and +# computes a checksum of these files that will become part of the hash. +failcache_key() { + local port=$1 + if [ -z "$port" ]; then + err "failcache_key expects a port argument, but none was given." + return 1 + fi + + local checksum + checksum=$(compute_failcache_hash "$port") + if [ $? -ne 0 ]; then + err "compute_failcache_hash $port failed" + return 2 + fi + + local canonical_variants + canonical_variants=$("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/canonical_variants.tcl" "$@") + if [ $? -ne 0 ]; then + err "tools/canonical_variants.tcl" "$@" "failed" + return 4 + fi + + echo "$port $canonical_variants $checksum" +} + +## Test whether a given port with variants has previously failed. +# +# Valid arguments are a port name, optionally followed by a variant +# specification. Succeeds if the port did not previsouly fail to build, fails +# if the port is known to fail. +failcache_test() { + local key + key=$(failcache_key "$@") + if [ $? -ne 0 ]; then + err "Could not determine failcache key for" "$@" + return 1 + fi + + test -f "${option_failcache_dir}/${key}" +} + +## Mark a build of a given port with variants as successful. +# +# Valid arguments are a port name, optionally followed by a variant +# specification. Removes any database entries that marked a port as failed. +failcache_success() { + local key + key=$(failcache_key "$@") + if [ $? -ne 0 ]; then + err "Could not determine failcache key for" "$@" + return 1 + fi + + rm -f "${option_failcache_dir}/${key}" +} + +## Mark a build of a given port with variants as failed. +# +# Valid arguments are a port name, optionally followed by a variant +# specification. Creates or updates the timestamp of a database entry that +# marks a port as failed. +failcache_failure() { + local key + key=$(failcache_key "$@") + if [ $? -ne 0 ]; then + err "Could not determine failcache key for" "$@" + return 1 + fi + + touch "${option_failcache_dir}/${key}" +} Index: mpbb =================================================================== --- mpbb (revision 152544) +++ mpbb (working copy) @@ -8,11 +8,9 @@ # Don't inherit any option variables from the calling environment. unset "${!option_@}" -# Print $0 and arguments to standard error. -# Unset IFS to ensure "$*" uses spaces as separators. -msg() (unset IFS; printf >&2 '%s: %s\n' "$0" "$*") -err() { msg error: "$@"; } -warn() { msg warning: "$@"; } +# Load function library +thisdir=$(cd "$(dirname "$0")" && pwd) +. "$thisdir/functions" mpbb-usage() { # "prog" is defined in mpbb-help. @@ -121,6 +119,7 @@ # Not really options, but pretend they are because they're global. option_jobs_dir=${option_work_dir}/infrastructure/jobs option_log_dir=${option_work_dir}/logs +option_failcache_dir=${option_work_dir}/failcache # Inform the user if old repositories are still present. if [[ -d ${option_work_dir}/tools/.svn ]]; then @@ -141,7 +140,6 @@ # must define functions "FOO" and "FOO-usage". cmds=() usages=(mpbb-usage) -thisdir=$(cd "$(dirname "$0")" && pwd) for cmdfile in "$thisdir/mpbb-"*; do # Unfortunately ShellCheck does not currently support following multiple # files, so we'll just disable the warning. Index: mpbb-install-dependencies =================================================================== --- mpbb-install-dependencies (revision 152544) +++ mpbb-install-dependencies (working copy) @@ -41,6 +41,9 @@ local dependencies local dependencies_count local dependencies_counter + local depname + local depvariants + local failcache_key # $option_log_dir is set in mpbb # shellcheck disable=SC2154 local log_status_dependencies="${option_log_dir}/dependencies-progress.txt" @@ -53,7 +56,7 @@ # calculate list of dependencies in-order # $option_prefix and $thisdir are set in mpbb # shellcheck disable=SC2154 - dependencies=$("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/dependencies.tcl" "$port") + dependencies=$("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/dependencies.tcl" "$@") if [ $? -ne 0 ]; then echo "Calculating dependencies for '$port' failed, aborting." >&2 echo "Building '$port' ... [ERROR] (failed to calculate dependencies) maintainers: $(get-maintainers "$port")." >> "$log_subports_progress" @@ -72,6 +75,24 @@ echo "$dependencies" | sed -E 's/^/ - /' | tee -a "$log_status_dependencies" echo >> "$log_status_dependencies" + # Check whether any of the dependencies have previously failed + echo "$dependencies" | while read -r dependency; do + # Split portname +variant1+variant2 into portname and variants, where + # the variants are optional. + depname=${dependency%% *} + depvariants=${dependency:${#depname}+1} + + # $depvariants isn't quoted on purpose + # shellcheck disable=SC2086 + if ! failcache_test "$depname" $depvariants; then + text="Dependency '${depname}' with variants '${depvariants}' has previously failed and is required." + echo "$text" >&2 + echo "$text" >> "$log_status_dependencies" + echo "Building '$port' ... [ERROR] (failed to install dependency '${depname}') maintainers: $(get-maintainers "$port" "${depname}")." >> "$log_subports_progress" + return 1 + fi + done + echo "$dependencies" | while read -r dependency; do # Split portname +variant1+variant2 into portname and variants, where # the variants are optional. @@ -84,12 +105,21 @@ # $depvariants isn't quoted on purpose # shellcheck disable=SC2086 if ! "${option_prefix}/bin/port" -d install --unrequested "$depname" $depvariants; then - echo "Build of dependency '${depname}' failed, aborting." >&2 + echo "Build of dependency '${depname}' with variants '${depvariants}' failed, aborting." >&2 echo "[FAIL]" >> "$log_status_dependencies" echo "Building '$port' ... [ERROR] (failed to install dependency '${depname}') maintainers: $(get-maintainers "$port" "${depname}")." >> "$log_subports_progress" + + # Update failcache + # $depvariants isn't quoted on purpose + # shellcheck disable=SC2086 + failcache_failure "$depname" $depvariants return 1 else echo "[OK]" >> "$log_status_dependencies" + # Remove failcache if it exists + # $depvariants isn't quoted on purpose + # shellcheck disable=SC2086 + failcache_success "$depname" $depvariants dependencies_counter=$((dependencies_counter + 1)) fi done Index: mpbb-install-port =================================================================== --- mpbb-install-port (revision 152544) +++ mpbb-install-port (working copy) @@ -55,10 +55,15 @@ time_start=$(date +%s) # $option_prefix is set in mpbb # shellcheck disable=SC2154 - if ! "${option_prefix}/bin/port" -dk install "$port"; then + if "${option_prefix}/bin/port" -dk install "$@"; then + # Remove failcache if it exists + failcache_success "$@" + else echo "Build of '$port' failed." # log: summary for the portwatcher echo "Building '$port' ... [ERROR] maintainers: $(get-maintainers "$port")." >> "$log_subports_progress" + # update failcache + failcache_failure "$@" return 1 fi time_stop=$(date +%s) Index: tools/canonical_variants.tcl =================================================================== --- tools/canonical_variants.tcl (nonexistent) +++ tools/canonical_variants.tcl (working copy) @@ -0,0 +1,97 @@ +#!/usr/bin/env port-tclsh +# -*- coding: utf-8; mode: tcl; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:ft=tcl:et:sw=4:ts=4:sts=4 +# Writes the canonical list of variants as it appears in binary archive names +# to stdout. +# +# Copyright (c) 2016 The MacPorts Project. +# Copyright (c) 2016 Clemens Lang <c...@macports.org> +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the MacPorts project, nor the names of any contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS +# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package require macports + +if {[llength $::argv] == 0} { + puts stderr "Usage: $argv0 <portname> \[(+|-)variant...\]" + exit 1 +} + +# initialize macports +if {[catch {mportinit "" "" ""} result]} { + ui_error "$errorInfo" + ui_error "Failed to initialize ports sytem: $result" + exit 1 +} + +# look up the path of the Portfile for the given port +set portname [lindex $::argv 0] +#try -pass_signal {...} +try { + set result [mportlookup $portname] + if {[llength $result] < 2} { + ui_error "No such port: $portname" + exit 1 + } +} catch {{*} eCode eMessage} { + ui_error "mportlookup $portname failed: $eMessage" + exit 1 +} + +# parse the given variants from the command line +array set variants {} +foreach item [lrange $::argv 1 end] { + foreach {_ sign variant} [regexp -all -inline -- {([-+])([[:alpha:]_]+[\w\.]*)} $item] { + set variants($variant) $sign + } +} + +# open the port to get its active variants +array set portinfo [lindex $result 1] +#try -pass_signal {...} +try { + set mport [mportopen $portinfo(porturl) [list subport $portname] [array get variants]] +} catch {{*} eCode eMessage} { + ui_error "mportopen ${portinfo(porturl)} failed: $eMessage" + exit 1 +} + +array set info [mportinfo $mport] +puts $info(canonical_active_variants) + +#try -pass_signal {...} +try { + mportclose $mport +} catch {{*} eCode eMessage} { + ui_warn "mportclose $portname failed: $eMessage" +} + +# shut down MacPorts +#try -pass_signal {...} +try { + mportshutdown +} catch {{*} eCode eMessage} { + ui_error "mportshutdown failed: $eMessage" + exit 1 +} Property changes on: tools/canonical_variants.tcl ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:executable ## -0,0 +1 ## +* \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +Id \ No newline at end of property _______________________________________________ macports-dev mailing list macports-dev@lists.macosforge.org https://lists.macosforge.org/mailman/listinfo/macports-dev