Faidon Liambotis has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/384791 )
Change subject: mirrors: stop shipping ftpsync, use package instead ...................................................................... mirrors: stop shipping ftpsync, use package instead ftpsync (src:archvsync) is now available as a package, albeit only in buster (and now, stretch-wikimedia). Switch to using that instead of shipping a copy in our puppet repository, thereby making updates more streamlined, especially in >= buster. Change-Id: I1cca0f806a014b5c05f7296b78eda7c2004c79a9 --- D modules/mirrors/files/archvsync/bin/common D modules/mirrors/files/archvsync/bin/ftpsync M modules/mirrors/files/ssh-debian-archvsync.pub M modules/mirrors/manifests/debian.pp M modules/mirrors/spec/classes/mirrors_debian_spec.rb M modules/mirrors/templates/ftpsync.conf.erb 6 files changed, 19 insertions(+), 1,191 deletions(-) Approvals: Faidon Liambotis: Looks good to me, approved jenkins-bot: Verified diff --git a/modules/mirrors/files/archvsync/bin/common b/modules/mirrors/files/archvsync/bin/common deleted file mode 100644 index f97016e..0000000 --- a/modules/mirrors/files/archvsync/bin/common +++ /dev/null @@ -1,292 +0,0 @@ -# -*- mode:sh -*- -# vim:syn=sh -# Little common functions - -# push a mirror attached to us. -# Arguments (using an array named SIGNAL_OPTS): -# -# $MIRROR - Name for the mirror, also basename for the logfile -# $HOSTNAME - Hostname to push to -# $USERNAME - Username there -# $SSHPROTO - Protocol version, either 1 or 2. -# $SSHKEY - the ssh private key file to use for this push -# $SSHOPTS - any other option ssh accepts, passed blindly, be careful -# $PUSHLOCKOWN - own lockfile name to touch after stage1 in pushtype=staged -# $PUSHTYPE - what kind of push should be done? -# all - normal, just push once with ssh backgrounded and finish -# staged - staged. first push stage1, then wait for $PUSHLOCKs to appear, -# then push stage2 -# $PUSHARCHIVE - what archive to sync? (Multiple mirrors behind one ssh key!) -# $PUSHCB - do we want a callback? -# $PUSHKIND - whats going on? are we doing mhop push or already stage2? -# $FROMFTPSYNC - set to true if we run from within ftpsync. -# -# This function assumes that the variable LOG is set to a directory where -# logfiles can be written to. -# Additionally $PUSHLOCKS has to be defined as a set of space delimited strings -# (list of "lock"files) to wait for if you want pushtype=staged -# -# Pushes might be done in background (for type all). -signal () { - ARGS="SIGNAL_OPTS[*]" - local ${!ARGS} - - MIRROR=${MIRROR:-""} - HOSTNAME=${HOSTNAME:-""} - USERNAME=${USERNAME:-""} - SSHPROTO=${SSHPROTO:-""} - SSHKEY=${SSHKEY:-""} - SSHOPTS=${SSHOPTS:-""} - PUSHLOCKOWN=${PUSHLOCKOWN:-""} - PUSHTYPE=${PUSHTYPE:-"all"} - PUSHARCHIVE=${PUSHARCHIVE:-""} - PUSHCB=${PUSHCB:-""} - PUSHKIND=${PUSHKIND:-"all"} - FROMFTPSYNC=${FROMFTPSYNC:-"false"} - - # And now get # back to space... - SSHOPTS=${SSHOPTS/\#/ } - - # Defaults we always want, no matter what - SSH_OPTIONS="-o user=${USERNAME} -o BatchMode=yes -o ServerAliveInterval=45 -o ConnectTimeout=45 -o PasswordAuthentication=no" - - # If there are userdefined ssh options, add them. - if [[ -n ${SSH_OPTS} ]]; then - SSH_OPTIONS="${SSH_OPTIONS} ${SSH_OPTS}" - fi - - # Does this machine need a special key? - if [[ -n ${SSHKEY} ]]; then - SSH_OPTIONS="${SSH_OPTIONS} -i ${SSHKEY}" - fi - - # Does this machine have an extra own set of ssh options? - if [[ -n ${SSHOPTS} ]]; then - SSH_OPTIONS="${SSH_OPTIONS} ${SSHOPTS}" - fi - - # Set the protocol version - if [[ ${SSHPROTO} -ne 1 ]] && [[ ${SSHPROTO} -ne 2 ]] && [[ ${SSHPROTO} -ne 99 ]]; then - # Idiots, we only want 1 or 2. Cant decide? Lets force 2. - SSHPROTO=2 - fi - - if [[ -n ${SSHPROTO} ]] && [[ ${SSHPROTO} -ne 99 ]]; then - SSH_OPTIONS="${SSH_OPTIONS} -${SSHPROTO}" - fi - - date -u >> "${LOGDIR}/${MIRROR}.log" - - PUSHARGS="" - # PUSHARCHIVE empty or not, we always add the sync:archive: command to transfer. - # Otherwise, if nothing else is added, ssh -f would not work ("no command to execute") - # But ftpsync does treat "sync:archive:" as the main archive, so this works nicely. - PUSHARGS="${PUSHARGS} sync:archive:${PUSHARCHIVE}" - - # We have a callback wish, tell downstreams - if [[ -n ${PUSHCB} ]]; then - PUSHARGS="${PUSHARGS} sync:callback" - fi - # If we are running an mhop push AND our downstream is one to receive it, tell it. - if [[ mhop = ${PUSHKIND} ]] && [[ mhop = ${PUSHTYPE} ]]; then - PUSHARGS="${PUSHARGS} sync:mhop" - fi - - if [[ all = ${PUSHTYPE} ]]; then - # Default normal "fire and forget" push. We background that, we do not care about the mirrors doings - log "Sending normal push" >> "${LOGDIR}/${MIRROR}.log" - PUSHARGS1="sync:all" - ssh -n $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 - if [[ $? -eq 255 ]]; then - error "Trigger to ${HOSTNAME} failed" >> "${LOG}" - else - log "Trigger to ${HOSTNAME} succeed" >> "${LOG}" - fi - elif [[ staged = ${PUSHTYPE} ]] || [[ mhop = ${PUSHTYPE} ]]; then - # Want a staged push. Fine, lets do that. Not backgrounded. We care about the mirrors doings. - log "Sending staged push" >> "${LOGDIR}/${MIRROR}.log" - - # Only send stage1 if we havent already send it. When called with stage2, we already did. - if [[ stage2 != ${PUSHKIND} ]]; then - # Step1: Do a push to only sync stage1, do not background - PUSHARGS1="sync:stage1" - ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 - if [[ $? -eq 255 ]]; then - error "Trigger to ${HOSTNAME} failed" >> "${LOG}" - else - log "Trigger to ${HOSTNAME} succeed" >> "${LOG}" - fi - touch "${PUSHLOCKOWN}" - - # Step2: Wait for all the other "lock"files to appear. - tries=0 - # We do not wait forever - while [[ ${tries} -lt ${PUSHDELAY} ]]; do - total=0 - found=0 - for file in ${PUSHLOCKS}; do - total=$(( total + 1 )) - if [[ -f ${file} ]]; then - found=$(( found + 1 )) - fi - done - if [[ ${total} -eq ${found} ]] || [[ -f ${LOCKDIR}/all_stage1 ]]; then - touch "${LOCKDIR}/all_stage1" - break - fi - tries=$(( tries + 5 )) - sleep 5 - done - # In case we did not have all PUSHLOCKS and still continued, note it - # This is a little racy, especially if the other parts decide to do this - # at the same time, but it wont hurt more than a mail too much, so I don't care much - if [[ ${tries} -ge ${PUSHDELAY} ]]; then - log "Failed to wait for all other mirrors. Failed ones are:" >> "${LOGDIR}/${MIRROR}.log" - for file in ${PUSHLOCKS}; do - if [[ ! -f ${file} ]]; then - log "${file}" >> "${LOGDIR}/${MIRROR}.log" - error "Missing Pushlockfile ${file} after waiting ${tries} second, continuing" - fi - done - fi - rm -f "${PUSHLOCKOWN}" - fi - - # Step3: It either timed out or we have all the "lock"files, do the rest - # If we are doing mhop AND are called from ftpsync - we now exit. - # That way we notify our uplink that we and all our clients are done with their - # stage1. It can then finish its own, and if all our upstreams downlinks are done, - # it will send us stage2. - # If we are not doing mhop or are not called from ftpsync, we start stage2 - if [[ true = ${FROMFTPSYNC} ]] && [[ mhop = ${PUSHKIND} ]]; then - return - else - PUSHARGS2="sync:stage2" - log "Now doing the second stage push" >> "${LOGDIR}/${MIRROR}.log" - ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS2}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 - if [[ $? -eq 255 ]]; then - error "Trigger to ${HOSTNAME} failed" >> "${LOG}" - else - log "Trigger to ${HOSTNAME} succeed" >> "${LOG}" - fi - fi - else - # Can't decide? Then you get nothing. - return - fi -} - -# callback, used by ftpsync -callback () { - # Defaults we always want, no matter what - SSH_OPTIONS="-o BatchMode=yes -o ServerAliveInterval=45 -o ConnectTimeout=45 -o PasswordAuthentication=no" - ssh $SSH_OPTIONS -i "$3" -o"user $1" "$2" callback:${HOSTNAME} -} - -# log something (basically echo it together with a timestamp) -# -# Set $PROGRAM to a string to have it added to the output. -log () { - if [[ -z "${PROGRAM}" ]]; then - echo "$(date +"%b %d %H:%M:%S") $(hostname -s) [$$] $@" - else - echo "$(date +"%b %d %H:%M:%S") $(hostname -s) ${PROGRAM}[$$]: $@" - fi -} - -# log the message using log() but then also send a mail -# to the address configured in MAILTO (if non-empty) -error () { - log "$@" - if [[ -n "${MAILTO}" ]]; then - echo "$@" | mail -e -s "[$PROGRAM@$(hostname -s)] ERROR [$$]" ${MAILTO} - fi -} - -# run a hook -# needs array variable HOOK setup with HOOKNR being a number an HOOKSCR -# the script to run. -hook () { - ARGS='HOOK[@]' - local "${!ARGS}" - if [[ -n ${HOOKSCR} ]]; then - log "Running hook $HOOKNR: ${HOOKSCR}" - set +e - ${HOOKSCR} - result=$? - set -e - if [[ ${result} -ne 0 ]] ; then - error "Back from hook $HOOKNR, got returncode ${result}" - else - log "Back from hook $HOOKNR, got returncode ${result}" - fi - return $result - else - return 0 - fi -} - -# Return the list of 2-stage mirrors. -get2stage() { - egrep '^(staged|mhop)' "${MIRRORS}" | { - while read MTYPE MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do - PUSHLOCKS="${LOCKDIR}/${MLNAME}.stage1 ${PUSHLOCKS}" - done - echo "$PUSHLOCKS" - } -} - -# Rotate logfiles -savelog() { - torotate="$1" - count=${2:-${LOGROTATE}} - while [[ ${count} -gt 0 ]]; do - prev=$(( count - 1 )) - if [[ -e ${torotate}.${prev} ]]; then - mv "${torotate}.${prev}" "${torotate}.${count}" - fi - count=$prev - done - if [[ -e ${torotate} ]]; then - mv "${torotate}" "${torotate}.0" - fi -} - -# Return rsync version -rsync_protocol() { - RSYNC_VERSION="$(${RSYNC} --version)" - RSYNC_REGEX="(protocol[ ]+version[ ]+([0-9]+))" - if [[ ${RSYNC_VERSION} =~ ${RSYNC_REGEX} ]]; then - echo ${BASH_REMATCH[2]} - fi - unset RSYNC_VERSION RSYNC_REGEX -} - -# Search config files in various locations -search_config() { - local file - if [ "$BASEDIR" ]; then - file="$BASEDIR/etc/$1" - if [ -f "$file" ]; then - echo "$file" - return - fi - fi - for i in ~/.config/archvsync /etc/archvsync; do - file="$i/$1" - if [ -f "$file" ]; then - echo "$file" - return - fi - done -} - -# Read config file -read_config() { - local config=$(search_config "$1") - if [ "$config" ]; then - . "$config" - return 0 - fi - return 1 -} diff --git a/modules/mirrors/files/archvsync/bin/ftpsync b/modules/mirrors/files/archvsync/bin/ftpsync deleted file mode 100755 index e3960fa..0000000 --- a/modules/mirrors/files/archvsync/bin/ftpsync +++ /dev/null @@ -1,857 +0,0 @@ -#! /bin/bash -# No, we can not deal with sh alone. - -set -e -set -u -# ERR traps should be inherited from functions too. (And command -# substitutions and subshells and whatnot, but for us the function is -# the important part here) -set -E - -# A pipeline's return status is the value of the last (rightmost) -# command to exit with a non-zero status, or zero if all commands exit -# success fully. -set -o pipefail - -# ftpsync script for Debian -# Based losely on a number of existing scripts, written by an -# unknown number of different people over the years. -# -# Copyright (C) 2008-2016 Joerg Jaspert <jo...@debian.org> -# Copyright (C) 2016 Peter Palfrader -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; version 2. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -# In case the admin somehow wants to have this script located someplace else, -# he can set BASEDIR, and we will take that. If it is unset we use $0 -# How the admin sets this isn't our place to deal with. One could use a wrapper -# for that. Or pam_env. Or whatever fits in the local setup. :) -BASEDIR=${BASEDIR:-"$(readlink -f $(dirname "$0")/..)"} - -# Script version. DO NOT CHANGE, *unless* you change the master copy maintained -# by Joerg Jaspert and the Debian mirroradm group. -# This is used to track which mirror is using which script version. -VERSION="20170204" - -# Source our common functions -. "${BASEDIR}/bin/common" - -######################################################################## -######################################################################## -## functions ## -######################################################################## -######################################################################## -# We want to be able to get told what kind of sync we should do. This -# might be anything, from the archive to sync, the stage to do, etc. A -# list of currently understood and valid options is below. Multiple -# options are seperated by space. All the words have to have the word -# sync: in front or nothing will get used! -# -# Option Behaviour -# stage1 Only do stage1 sync -# stage2 Only do stage2 sync -# all Do a complete sync -# mhop Do a mhop sync, usually additionally to stage1 -# archive:foo Sync archive foo (if config for foo is available) -# callback Call back when done (needs proper ssh setup for this to -# work). It will always use the "command" callback:$HOSTNAME -# where $HOSTNAME is the one defined below/in config and -# will happen before slave mirrors are triggered. -# -# So to get us to sync all of the archive behind bpo and call back when -# we are done, a trigger command of -# "ssh $USER@$HOST sync:all sync:archive:bpo sync:callback" will do the -# trick. -check_commandline() { - while [[ $# -gt 0 ]]; do - case "$1" in - sync:stage1) - SYNCSTAGE1="true" - SYNCALL="false" - ;; - sync:stage2) - SYNCSTAGE2="true" - SYNCALL="false" - ;; - sync:callback) - SYNCCALLBACK="true" - ;; - sync:archive:*) - ARCHIVE=${1##sync:archive:} - # We do not like / or . in the remotely supplied archive name. - ARCHIVE=${ARCHIVE//\/} - ARCHIVE=${ARCHIVE//.} - ;; - sync:all) - SYNCALL="true" - ;; - sync:mhop) - SYNCMHOP="true" - ;; - *) - echo "Unknown option ${1} ignored" - ;; - esac - shift # Check next set of parameters. - done -} - -# All the stuff we want to do when we exit, no matter where -cleanup() { - trap - ERR TERM HUP INT QUIT EXIT - # all done. Mail the log, exit. - log "Mirrorsync done"; - - if [[ -n ${MAILTO} ]]; then - # In case rsync had something on stderr - if [[ -s ${LOGDIR}/rsync-${NAME}.error ]]; then - mail -E -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error" - fi - if [[ ${ERRORSONLY} = false ]]; then - # And the normal log - MAILFILES="${LOG}" - if [[ ${FULLLOGS} = true ]]; then - # Someone wants full logs including rsync - MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log" - fi - cat ${MAILFILES} | mail -E -s "[${PROGRAM}@$(hostname -s)] archive sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} - fi - fi - - savelog "${LOGDIR}/rsync-${NAME}.log" - savelog "${LOGDIR}/rsync-${NAME}.error" - savelog "$LOG" > /dev/null - - rm -f "${LOCK}" -} - -# Check rsyncs return value -check_rsync() { - ret=$1 - msg=$2 - - # Lets get a statistical value - SPEED="unknown" - if [[ -f ${LOGDIR}/rsync-${NAME}.log ]]; then - SPEED=$( - SPEEDLINE=$(egrep '[0-9.,]+ bytes/sec' "${LOGDIR}/rsync-${NAME}.log" | tail -n 1) - set "nothing" ${SPEEDLINE} - echo ${8:-""} - ) - if [[ -n ${SPEED} ]]; then - SPEED=${SPEED%%.*} - SPEED=${SPEED//,} - SPEED=$(( $SPEED / 1024 )) - fi - fi - log "Latest recorded rsync transfer speed: ${SPEED} KB/s" - - # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED - # and us re-running. If it's not, uplink is broken anyways. - case "${ret}" in - 0) return 0;; - 24) return 0;; - 23) return 2;; - 30) return 2;; - *) - error "ERROR: ${msg}" - return 1 - ;; - esac -} - -# Write a tracefile -tracefile() { - local TRACEFILE=${1:-"${TO}/${TRACE}"} - - case ${EXTENDEDTRACE} in - none) - log "No trace file wanted. Not creating one" - ;; - touch) - log "Just touching the trace file" - touch "${TRACEFILE}" - ;; - date|terse|full) - log "Creating a ${EXTENDEDTRACE} trace file" - tracedir=$(dirname "${TRACEFILE}") - if [[ -d ${tracedir} ]]; then - { - LC_ALL=POSIX LANG=POSIX date -u - if [[ date != ${EXTENDEDTRACE} ]]; then - rfc822date=$(LC_ALL=POSIX LANG=POSIX date -u -R) - echo "Date: ${rfc822date}" - echo "Date-Started: ${DATE_STARTED}" - - FILESOA=$(awk -F": " ' $1=="Archive serial" {print $2}' ${TO}/project/trace/master || echo unknown ) - echo "Archive serial: ${FILESOA}" - - echo "Used ftpsync version: ${VERSION}" - echo "Running on host: ${TRACEHOST}" - if [[ full = ${EXTENDEDTRACE} ]]; then - ARCH=$((find ${TO}/dists -name Release -exec sed -ne 's/^Architecture: //p' '{}' + 2>/dev/null || echo "unknown") | sort -u | tr '\n' ' ') - echo "Architectures: ${ARCH}" - if [[ ${ARCH_INCLUDE} ]]; then - echo "Architectures-Configuration: INCLUDE $(tr ' ' '\n' <<< ${ARCH_INCLUDE} | sort -u | tr '\n' ' ')" - elif [[ ${ARCH_EXCLUDE} ]]; then - echo "Architectures-Configuration: EXCLUDE $(tr ' ' '\n' <<< ${ARCH_EXCLUDE} | sort -u | tr '\n' ' ')" - else - echo "Architectures-Configuration: ALL" - fi - echo "Upstream-mirror: ${RSYNC_HOST}" - echo "SSL: ${RSYNC_SSL}" - total=0 - if [[ -e ${LOGDIR}/rsync-${NAME}.log ]]; then - for bytes in $(awk -F': ' '$1 == "Total bytes received" {print $2} ' "${LOGDIR}/rsync-${NAME}.log"); do - bytes=${bytes//,} - total=$(( total + bytes )) - done - echo "Total bytes received in rsync: ${total}" - fi - total_time=$(( STATS_TOTAL_RSYNC_TIME1 + STATS_TOTAL_RSYNC_TIME2 )) - echo "Total time spent in stage1 rsync: ${STATS_TOTAL_RSYNC_TIME1}" - echo "Total time spent in stage2 rsync: ${STATS_TOTAL_RSYNC_TIME2}" - echo "Total time spent in rsync: ${total_time}" - if [[ 0 != ${total_time} ]]; then - rate=$(( total / total_time )) - echo "Average rate: ${rate} B/s" - fi - fi # full trace - fi # other traces - } > "${TRACEFILE}.new" - mv "${TRACEFILE}.new" "${TRACEFILE}" - - if [[ full = ${EXTENDEDTRACE} ]] || [[ -e ${TO}/${TRACEHIERARCHY}.mirror ]]; then - { - if [[ -e ${TO}/${TRACEHIERARCHY}.mirror ]]; then - cat ${TO}/${TRACEHIERARCHY}.mirror - fi - echo "$(basename "${TRACEFILE}") ${MIRRORNAME} ${TRACEHOST} ${RSYNC_HOST}" - } > "${TO}/${TRACEHIERARCHY}".new - mv "${TO}/${TRACEHIERARCHY}".new "${TO}/${TRACEHIERARCHY}" - cp "${TO}/${TRACEHIERARCHY}" "${TO}/${TRACEHIERARCHY}.mirror" - - (cd ${tracedir} && ls -1rt) | grep -E -v '^_|^master$' > "${TO}/${TRACELIST}" - else - rm -f "${TO}/${TRACEHIERARCHY}" "${TO}/${TRACELIST}" - fi - fi - ;; - *) - error "Unsupported EXTENDEDTRACE value configured in ${BASEDIR}/etc/${NAME}.conf, please fix" - ;; - esac - -} - -arch_imexclude() { - local param="$1" arch="$2" - EXCLUDE="${EXCLUDE} --${param}=/dists/**/binary-${arch}/ --${param}=/dists/**/installer-${arch}/ --${param}=/dists/**/Contents-${arch}.gz --${param}=/dists/**/Contents-udeb-${arch}.gz --${param}=/dists/**/Contents-${arch}.diff/ --${param}=/indices/**/arch-${arch}.files --${param}=/indices/**/arch-${arch}.list.gz --${param}=/pool/**/*_${arch}.deb --${param}=/pool/**/*_${arch}.udeb --${param}=/pool/**/*_${arch}.changes" -} - -arch_exclude() { - arch_imexclude exclude "$1" -} - -arch_include() { - arch_imexclude include "$1" -} - -# Learn which archs to include/exclude based on ARCH_EXCLUDE and ARCH_INCLUDE -# settings. -# -# Updates SOURCE_EXCLUDE (if not set already), and sets -# EXCLUDE (which might also have --include statementss -# followed by a --exclude *_*.<things>. -set_exclude_include_archs() { - local exclude_sources=false - - if [[ -n "${ARCH_EXCLUDE}" ]] && [[ -n "${ARCH_INCLUDE}" ]]; then - echo >&2 "ARCH_EXCLUDE and ARCH_INCLUDE are mutually exclusive. Set only one." - exit 1 - fi - - if [[ -n "${ARCH_EXCLUDE}" ]]; then - for ARCH in ${ARCH_EXCLUDE}; do - arch_exclude ${ARCH} - if [[ ${ARCH} = source ]]; then - exclude_sources=true - fi - done - elif [[ -n "${ARCH_INCLUDE}" ]]; then - local include_arch_all=false - exclude_sources=true - for ARCH in ${ARCH_INCLUDE}; do - arch_include ${ARCH} - if [[ ${ARCH} = source ]]; then - exclude_sources=false - else - include_arch_all=true - fi - done - if [[ true = ${include_arch_all} ]]; then - arch_include all - fi - arch_exclude '*' - fi - - # we do not want sources, and SOURCE_EXCLUDE is not set in the config yet. - if [[ true = ${exclude_sources} ]] && [[ -z ${SOURCE_EXCLUDE} ]]; then - SOURCE_EXCLUDE=" --exclude=/dists/**/source/ --exclude=/pool/**/*.tar.* --exclude=/pool/**/*.diff.* --exclude=/pool/**/*.dsc " - fi -} - -######################################################################## -######################################################################## - - -# As what are we called? -NAME="$(basename $0)" -# The original command line arguments need to be saved! -if [[ $# -gt 0 ]]; then - ORIGINAL_COMMAND=$* -else - ORIGINAL_COMMAND="" -fi - -SSH_ORIGINAL_COMMAND=${SSH_ORIGINAL_COMMAND:-""} -# Now, check if we got told about stuff via ssh -if [[ -n ${SSH_ORIGINAL_COMMAND} ]]; then - # We deliberately add "nothing" and ignore it right again, to avoid - # people from outside putting some set options in the first place, - # making us parse them... - set "nothing" "${SSH_ORIGINAL_COMMAND}" - shift - # Yes, unqouted $* here. Or the function will only see it as one - # parameter, which doesnt help the case in it. - check_commandline $* -fi - -# Now, we can locally override all the above variables by just putting -# them into the .ssh/authorized_keys file forced command. -if [[ -n ${ORIGINAL_COMMAND} ]]; then - set ${ORIGINAL_COMMAND} - check_commandline $* -fi - -# If we have been told to do stuff for a different archive than default, -# set the name accordingly. -ARCHIVE=${ARCHIVE:-""} -if [[ -n ${ARCHIVE} ]]; then - NAME="${NAME}-${ARCHIVE}" -fi - -# Now source the config for the archive we run on. -# (Yes, people can also overwrite the options above in the config file -# if they want to) -if ! read_config "${NAME}.conf"; then - echo "Nono, you can't tell us about random archives. Bad boy!" - exit 1 -fi - -######################################################################## -# There should be nothing to edit here, use the config file # -######################################################################## -MIRRORNAME=${MIRRORNAME:-$(hostname -f)} -# Where to put logfiles in -LOGDIR=${LOGDIR:-"${BASEDIR}/log"} -# Our own logfile -LOG=${LOG:-"${LOGDIR}/${NAME}.log"} - -# Where should we put all the mirrored files? -TO=${TO:-"/srv/mirrors/debian/"} - -# used by log() and error() -PROGRAM=${PROGRAM:-"${NAME}"} - -# Where to send mails about mirroring to? -if [[ $(hostname -d) != debian.org ]]; then - # We are not on a debian.org host - MAILTO=${MAILTO:-"root"} -else - # Yay, on a .debian.org host - MAILTO=${MAILTO:-"mirrorl...@debian.org"} -fi -# Want errors only or every log? -ERRORSONLY=${ERRORSONLY:-"true"} -# Want full logs, ie. including the rsync one? -FULLLOGS=${FULLLOGS:-"false"} - -# How many logfiles to keep -LOGROTATE=${LOGROTATE:-14} - -# Our lockfile -LOCK=${LOCK:-"${TO}/Archive-Update-in-Progress-${MIRRORNAME}"} -# timeout for the lockfile, in case we have bash older than v4 (and no /proc) -LOCKTIMEOUT=${LOCKTIMEOUT:-3600} -# sleeping time when an AUIP file is found but is not ours -UIPSLEEP=${UIPSLEEP:-1200} -# retries whenever an upstream (or possibly stale) AUIP file is found -UIPRETRIES=${UIPRETRIES:-3} -# Do we need another rsync run? -UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}" -# Trace file for mirror stats and checks (make sure we get full hostname) -TRACE=${TRACE:-"project/trace/${MIRRORNAME}"} -TRACEHIERARCHY=${TRACEHIERARCHY:-"project/trace/_hierarchy"} -TRACELIST=${TRACELIST:-"project/trace/_traces"} -# The trace file can have different format/contents. Here you can select -# what it will be. -# Possible values are -# "full" - all information -# "terse" - partial, ftpsync version and local hostname -# "date" - basic, timestamp only (date -u) -# "touch" - just touch the file in existance -# "none" - no tracefile at all -# -# Default and required value for Debian mirrors is full. -EXTENDEDTRACE=${EXTENDEDTRACE:-"full"} - -# The local hostname to be written to the trace file. -TRACEHOST=${TRACEHOST:-$(hostname -f)} - -# rsync program -RSYNC=${RSYNC:-rsync} -# Rsync filter rules. Used to protect various files we always want to keep, even if we otherwise delete -# excluded files -RSYNC_FILTER=${RSYNC_FILTER:-"--filter=protect_Archive-Update-in-Progress-${MIRRORNAME} --filter=protect_${TRACE} --filter=protect_${TRACE}-stage1 --exclude=${TRACEHIERARCHY} --filter=protect_${TRACEHIERARCHY} --exclude=${TRACELIST} --filter=protect_${TRACELIST} --filter=protect_Archive-Update-Required-${MIRRORNAME}"} -# limit I/O bandwidth. Value is KBytes per second, unset or 0 is unlimited -RSYNC_BW=${RSYNC_BW:-0} -RSYNC_PROTOCOL=$(rsync_protocol) - -# Set the delete method to --delete-delay if protocol version is 30 or -# greater (meaning rsync 3.0.0 or greater is used). Use --delete-after -# otherwise. -if [[ 30 -le $RSYNC_PROTOCOL ]]; then - RSYNC_DELETE_METHOD=delay -else - RSYNC_DELETE_METHOD=after -fi - -# Default rsync options for *every* rsync call -RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-prltvHSB8192 --timeout 3600 --stats ${RSYNC_FILTER}"} -# Options we only use in the first pass, where we do not want packages/sources to fly in yet and don't want to delete files -RSYNC_OPTIONS1=${RSYNC_OPTIONS1:-"--exclude=Packages* --exclude=Sources* --exclude=Release* --exclude=InRelease --include=i18n/by-hash/** --exclude=i18n/* --exclude=ls-lR*"} -# Options for the second pass, where we do want everything, including deletion of old and now unused files -RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-excluded"} -# Which rsync share to use on our upstream mirror? -RSYNC_PATH=${RSYNC_PATH:-"debian"} - -# Extra rsync options as defined by the admin locally. Won't be set -# to any default by ftpsync. Those will be added to EACH AND EVERY rsync call. -RSYNC_EXTRA=${RSYNC_EXTRA:-""} - -# Now add the bwlimit option. As default is 0 we always add it, rsync interprets -# 0 as unlimited, so this is safe. -RSYNC_OPTIONS="${RSYNC_EXTRA} --bwlimit=${RSYNC_BW} ${RSYNC_OPTIONS}" - -# Finally, make sure RSYNC_OPTIONS2 has either --delete-after or --delete-delay -RSYNC_OPTION_REGEX="--delete-(after|delay)" -if ! [[ ${RSYNC_OPTIONS2} =~ ${RSYNC_OPTION_REGEX} ]]; then - RSYNC_OPTIONS2+=" --delete-${RSYNC_DELETE_METHOD}" -fi -unset RSYNC_OPTION_REGEX - -# We have no default host to sync from, but will error out if its unset -RSYNC_HOST=${RSYNC_HOST:-""} -# Error out if we have no host to sync from -if [[ -z ${RSYNC_HOST} ]]; then - error "Missing a host to mirror from, please set RSYNC_HOST variable in ${BASEDIR}/etc/${NAME}.conf" -fi - -# our username for the rsync share -RSYNC_USER=${RSYNC_USER:-""} -# the password -RSYNC_PASSWORD=${RSYNC_PASSWORD:-""} - -# a possible proxy -RSYNC_PROXY=${RSYNC_PROXY:-""} - -# Do we sync stage1? -SYNCSTAGE1=${SYNCSTAGE1:-"false"} -# Do we sync stage2? -SYNCSTAGE2=${SYNCSTAGE2:-"false"} -# Do we sync all? -SYNCALL=${SYNCALL:-"true"} -# Do we have a mhop sync? -SYNCMHOP=${SYNCMHOP:-"false"} -# Do we callback? (May get changed later) -SYNCCALLBACK=${SYNCCALLBACK:-"false"} -# If we call back we need some more options defined in the config file. -CALLBACKUSER=${CALLBACKUSER:-"archvsync"} -CALLBACKHOST=${CALLBACKHOST:-"none"} -CALLBACKKEY=${CALLBACKKEY:-"none"} - -# General excludes. Don't list architecture specific stuff here, use ARCH_EXCLUDE for that! -EXCLUDE=${EXCLUDE:-""} - -# collect some stats -STATS_TOTAL_RSYNC_TIME1=0 -STATS_TOTAL_RSYNC_TIME2=0 - -# The temp directory used by rsync --delay-updates is not -# world-readable remotely. Always exclude it to avoid errors. -EXCLUDE="${EXCLUDE} --exclude=.~tmp~/" - -SOURCE_EXCLUDE=${SOURCE_EXCLUDE:-""} -ARCH_EXCLUDE=${ARCH_EXCLUDE:-""} -ARCH_INCLUDE=${ARCH_INCLUDE:-""} - -RSYNC_SSL=${RSYNC_SSL:-"false"} -RSYNC_SSL_PORT=${RSYNC_SSL_PORT:-"1873"} -RSYNC_SSL_CAPATH=${RSYNC_SSL_CAPATH:-"/etc/ssl/certs"} -RSYNC_SSL_METHOD=${RSYNC_SSL_METHOD:-"stunnel4"} - -if [[ true != ${RSYNC_SSL} ]]; then - RSYNC_SSL_OPTIONS="" -else - export RSYNC_SSL_PORT - export RSYNC_SSL_CAPATH - export RSYNC_SSL_METHOD - RSYNC_SSL_OPTIONS="-e ${BASEDIR}/bin/rsync-ssl-tunnel" -fi - -# Hooks -HOOK1=${HOOK1:-""} -HOOK2=${HOOK2:-""} -HOOK3=${HOOK3:-""} -HOOK4=${HOOK4:-""} -HOOK5=${HOOK5:-""} - -# Are we a hub? -HUB=${HUB:-"false"} - -######################################################################## -# Really nothing to see below here. Only code follows. # -######################################################################## -######################################################################## -DATE_STARTED=$(LC_ALL=POSIX LANG=POSIX date -u -R) - -# Exclude architectures defined in $ARCH_EXCLUDE -set_exclude_include_archs - -# Some sane defaults -cd "${BASEDIR}" -umask 022 - -# If we are here for the first time, create the -# destination and the trace directory -mkdir -p "${TO}/project/trace" - -# Used to make sure we will have the archive fully and completly synced before -# we stop, even if we get multiple pushes while this script is running. -# Otherwise we can end up with a half-synced archive: -# - get a push -# - sync, while locked -# - get another push. Of course no extra sync run then happens, we are locked. -# - done. Archive not correctly synced, we don't have all the changes from the second push. -touch "${UPDATEREQUIRED}" - -# Check to see if another sync is in progress -if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then - if [[ ${BASH_VERSINFO[0]} -gt 3 ]] || [[ -L /proc/self ]]; then - # We have a recent enough bash version, lets do it the easy way, - # the lock will contain the right pid, thanks to $BASHPID - if ! $(kill -0 $(< ${LOCK}) 2>/dev/null); then - # Process does either not exist or is not owned by us. - echo "$$" > "${LOCK}" - else - echo "Unable to start rsync, lock file still exists, PID $(< ${LOCK})" - exit 1 - fi - else - # Old bash, means we dont have the right pid in our lockfile - # So take a different way - guess if it is still there by comparing its age. - # Not optimal, but hey. - stamptime=$(date --reference="${LOCK}" +%s) - unixtime=$(date +%s) - difference=$(( $unixtime - $stamptime )) - if [[ ${difference} -ge ${LOCKTIMEOUT} ]]; then - # Took longer than LOCKTIMEOUT minutes? Assume it broke and take the lock - echo "$$" > "${LOCK}" - else - echo "Unable to start rsync, lock file younger than one hour" - exit 1 - fi - fi -fi - -# When we exit normally we call cleanup on our own. Otherwise we want it called by -# this trap. (We can not trap on EXIT, because that is called when the main script -# exits. Which also happens when we background the mainroutine, ie. while we still -# run!) -trap cleanup ERR TERM HUP INT QUIT - -# Start log by redirecting stdout and stderr there and closing stdin -mkdir -p "$LOGDIR" -exec >"$LOG" 2>&1 <&- -log "Mirrorsync start" - -# Look who pushed us and note that in the log. -SSH_CONNECTION=${SSH_CONNECTION:-""} -PUSHFROM="${SSH_CONNECTION%%\ *}" -if [[ -n ${PUSHFROM} ]]; then - log "We got pushed from ${PUSHFROM}" -fi - -if [[ true = ${SYNCCALLBACK} ]]; then - if [[ none = ${CALLBACKHOST} ]] || [[ none = ${CALLBACKKEY} ]]; then - SYNCCALLBACK="false" - error "We are asked to call back, but we do not know where to and do not have a key, ignoring callback" - fi -fi - -HOOK=( - HOOKNR=1 - HOOKSCR=${HOOK1} -) -hook $HOOK - -# Now, we might want to sync from anonymous too. -# This is that deep in this script so hook1 could, if wanted, change things! -if [[ -z ${RSYNC_USER} ]]; then - RSYNCPTH="${RSYNC_HOST}" -else - RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}" -fi - -# Now do the actual mirroring, and run as long as we have an updaterequired file. -export RSYNC_PASSWORD -export RSYNC_PROXY - -UPDATE_RETRIES=0 - -while [[ -e ${UPDATEREQUIRED} ]]; do - log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists" - - # if we want stage1 *or* all - if [[ true = ${SYNCSTAGE1} ]] || [[ true = ${SYNCALL} ]]; then - while [[ -e ${UPDATEREQUIRED} ]]; do - rm -f "${UPDATEREQUIRED}" - log "Running stage1: ${RSYNC} ${RSYNC_SSL_OPTIONS} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" - - set +e - # Step one, sync everything except Packages/Releases - rsync_started=$(date +%s) - ${RSYNC} ${RSYNC_SSL_OPTIONS} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} \ - ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>"${LOGDIR}/rsync-${NAME}.log" 2>>"${LOGDIR}/rsync-${NAME}.error" - result=$? - rsync_ended=$(date +%s) - STATS_TOTAL_RSYNC_TIME1=$(( STATS_TOTAL_RSYNC_TIME1 + rsync_ended - rsync_started )) - set -e - - log "Back from rsync with returncode ${result}" - done - else - # Fake a good resultcode - result=0 - fi # Sync stage 1? - rm -f "${UPDATEREQUIRED}" - - set +e - check_rsync $result "Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}" - GO=$? - set -e - if [[ ${GO} -eq 2 ]] && [[ -e ${UPDATEREQUIRED} ]]; then - log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" - elif [[ ${GO} -ne 0 ]]; then - exit 3 - fi - - HOOK=( - HOOKNR=2 - HOOKSCR=${HOOK2} - ) - hook $HOOK - - # if we want stage2 *or* all - if [[ true = ${SYNCSTAGE2} ]] || [[ true = ${SYNCALL} ]]; then - upstream_uip=false - for aupfile in "${TO}/Archive-Update-in-Progress-"*; do - case "$aupfile" in - "${TO}/Archive-Update-in-Progress-*") - error "Lock file is missing, this should not happen" - ;; - "${LOCK}") - : - ;; - *) - if [[ -f $aupfile ]]; then - # Remove the file, it will be synced again if - # upstream is still not done - rm -f "$aupfile" - else - log "AUIP file '$aupfile' is not really a file, weird" - fi - upstream_uip=true - ;; - esac - done - - if [[ true = ${upstream_uip} ]]; then - log "Upstream archive update in progress, skipping stage2" - if [[ ${UPDATE_RETRIES} -lt ${UIPRETRIES} ]]; then - log "Retrying update in ${UIPSLEEP}" - touch "${UPDATEREQUIRED}" - UPDATE_RETRIES=$(($UPDATE_RETRIES+1)) - sleep "${UIPSLEEP}" - result=0 - else - error "Update has been retried ${UPDATE_RETRIES} times, aborting" - log "Perhaps upstream is still updating or there's a stale AUIP file" - result=1 - fi - else - log "Running stage2: ${RSYNC} ${RSYNC_SSL_OPTIONS} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" - - set +e - # We are lucky, it worked. Now do step 2 and sync again, this time including - # the packages/releases files - rsync_started=$(date +%s) - ${RSYNC} ${RSYNC_SSL_OPTIONS} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \ - ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>"${LOGDIR}/rsync-${NAME}.log" 2>>"${LOGDIR}/rsync-${NAME}.error" - result=$? - rsync_ended=$(date +%s) - STATS_TOTAL_RSYNC_TIME2=$(( STATS_TOTAL_RSYNC_TIME2 + rsync_ended - rsync_started )) - set -e - - log "Back from rsync with returncode ${result}" - fi - else - # Fake a good resultcode - result=0 - fi # Sync stage 2? - - set +e - check_rsync $result "Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}" - GO=$? - set -e - if [[ ${GO} -eq 2 ]] && [[ -e ${UPDATEREQUIRED} ]]; then - log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" - elif [[ ${GO} -ne 0 ]]; then - exit 4 - fi - - HOOK=( - HOOKNR=3 - HOOKSCR=${HOOK3} - ) - hook $HOOK -done - -# We only update our tracefile when we had a stage2 or an all sync. -# Otherwise we would update it after stage1 already, which is wrong. -if [[ true = ${SYNCSTAGE2} ]] || [[ true = ${SYNCALL} ]]; then - tracefile - if [[ true = ${SYNCALL} ]]; then - rm -f "${TO}/${TRACE}-stage1" - fi -elif [[ true = ${SYNCSTAGE1} ]]; then - tracefile "${TO}/${TRACE}-stage1" -fi - - -HOOK=( - HOOKNR=4 - HOOKSCR=${HOOK4} -) -hook $HOOK - -if [[ true = ${SYNCCALLBACK} ]]; then - set +e - callback ${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}" - set -e -fi - -# Remove the Archive-Update-in-Progress file before we push our downstreams. -rm -f "${LOCK}" - -# Check if there is a newer version of ftpsync. If so inform the admin, but not -# more than once every third day. -if [[ -r ${TO}/project/ftpsync/LATEST.VERSION ]]; then - LATEST=$(< "${TO}/project/ftpsync/LATEST.VERSION") - if ! [[ ${LATEST} =~ [0-9]+ ]]; then - LATEST=0 - fi - if [[ ${LATEST} -gt ${VERSION} ]]; then - if [[ -n ${MAILTO} ]]; then - difference=0 - if [[ -f ${LOGDIR}/ftpsync.newversion ]]; then - stamptime=$(< "${LOGDIR}/ftpsync.newversion") - unixtime=$(date +%s) - difference=$(( $unixtime - $stamptime )) - fi - if [[ ${difference} -ge 259200 ]]; then - # Only warn every third day - mail -s "[$(hostname -s)] Update for ftpsync available" ${MAILTO} <<EOF -Hello admin, - -i found that there is a new version of me available. -Me lonely ftpsync is currently version: ${VERSION} -New release of myself is available as: ${LATEST} - -Me, myself and I - and the Debian mirroradmins - would be very grateful -if you could update me. You can find the latest version on your mirror, -check $(hostname -s):${TO}/project/ftpsync/ftpsync-${LATEST}.tar.gz - -You can ensure the validity of that file by using sha512sum or md5sum -against the available checksum files secured with a signature from the -Debian FTPMaster signing key. - -EOF - - date +%s > "${LOGDIR}/ftpsync.newversion" - fi - fi - else - # Remove a possible stampfile - rm -f "${LOGDIR}/ftpsync.newversion" - fi -fi - -if [[ ${HUB} = true ]]; then - # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop - if [[ true = ${SYNCSTAGE2} ]] || [[ true = ${SYNCALL} ]] || [[ true = ${SYNCMHOP} ]]; then - RUNMIRRORARGS="" - if [[ -n ${ARCHIVE} ]]; then - # We tell runmirrors about the archive we are running on. - RUNMIRRORARGS="-a ${ARCHIVE}" - fi - # We also tell runmirrors that we are running it from within ftpsync, so it can change - # the way it works with mhop based on that. - RUNMIRRORARGS="${RUNMIRRORARGS} -f" - - if [[ true = ${SYNCSTAGE1} ]]; then - # This is true when we have a mhop sync. A normal multi-stage push sending stage1 will - # not get to this point. - # So if that happens, tell runmirrors we are doing mhop - RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop" - elif [[ true = ${SYNCSTAGE2} ]]; then - RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2" - elif [[ true = ${SYNCALL} ]]; then - RUNMIRRORARGS="${RUNMIRRORARGS} -k all" - fi - log "Trigger slave mirrors using ${RUNMIRRORARGS}" - ${BASEDIR}/bin/runmirrors ${RUNMIRRORARGS} - log "Trigger slave done" - - HOOK=( - HOOKNR=5 - HOOKSCR=${HOOK5} - ) - hook $HOOK - fi -fi - -# All done, lets call cleanup -cleanup diff --git a/modules/mirrors/files/ssh-debian-archvsync.pub b/modules/mirrors/files/ssh-debian-archvsync.pub index 19b2a70..ae92c21 100644 --- a/modules/mirrors/files/ssh-debian-archvsync.pub +++ b/modules/mirrors/files/ssh-debian-archvsync.pub @@ -1 +1 @@ -no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,no-user-rc,command="/var/lib/mirror/archvsync/bin/ftpsync" ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDNZoV9j9KFMFssqLLbKF0b35npFAE3USHgtyN3pUsFm8Wsh1i4ZVly+8Nks7LVJrAI8uCrfvZKyQrqdcjAyy7UAGXfz5YrMxj0PZbvz9rHWa7Hxl+wO5mYwtklN8ek7uFgqDCmMOkE+4uIxOlPzDJ95BUBx5SVm8jAaMmaBoX20rxO5BGvguVU4RJ1V9EZKB4B9pGbdScRJOWAGUrg8HSKJGtlNQTF0xIU6P6quF0O+qpjF9CiB2GVzRJZXNpAhlekpVx5/Xg6N+sIN7QxxHViO7xwTQgteUehbrpToNc1N4jf4kciHmP/jdgEus/W61U410QfXVUYKOTDuGzAotl/ archvs...@syncproxy.cna.debian.org (aka mirror-umn.debian.org; 2015-12-09) +no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,no-user-rc,command="/usr/bin/ftpsync" ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDNZoV9j9KFMFssqLLbKF0b35npFAE3USHgtyN3pUsFm8Wsh1i4ZVly+8Nks7LVJrAI8uCrfvZKyQrqdcjAyy7UAGXfz5YrMxj0PZbvz9rHWa7Hxl+wO5mYwtklN8ek7uFgqDCmMOkE+4uIxOlPzDJ95BUBx5SVm8jAaMmaBoX20rxO5BGvguVU4RJ1V9EZKB4B9pGbdScRJOWAGUrg8HSKJGtlNQTF0xIU6P6quF0O+qpjF9CiB2GVzRJZXNpAhlekpVx5/Xg6N+sIN7QxxHViO7xwTQgteUehbrpToNc1N4jf4kciHmP/jdgEus/W61U410QfXVUYKOTDuGzAotl/ archvs...@syncproxy.cna.debian.org (aka mirror-umn.debian.org; 2015-12-09) diff --git a/modules/mirrors/manifests/debian.pp b/modules/mirrors/manifests/debian.pp index 3b72275..ff5ce64 100644 --- a/modules/mirrors/manifests/debian.pp +++ b/modules/mirrors/manifests/debian.pp @@ -23,38 +23,12 @@ mode => '0755', } - # this is <https://ftp-master.debian.org/git/archvsync.git> - # right now we just ship bin/ftpsync & bin/common - # there is soon going to be a Debian package, use that then instead - file { "${mirrors::homedir}/archvsync": - ensure => directory, - recurse => true, - purge => true, - owner => 'mirror', - group => 'mirror', - mode => '0755', - source => 'puppet:///modules/mirrors/archvsync', - } - - # don't purge logs (you'd expect more from people that love the FHS...) - file { "${mirrors::homedir}/archvsync/log": - ensure => directory, - purge => false, - owner => 'mirror', - group => 'mirror', - mode => '0755', - } - - # move to .config/archvsync or /etc/archvsync? - file { "${mirrors::homedir}/archvsync/etc": - ensure => directory, - owner => 'mirror', - group => 'mirror', - mode => '0755', + package { 'ftpsync': + ensure => present, } # this is our configuration for archvsync - file { "${mirrors::homedir}/archvsync/etc/ftpsync.conf": + file { '/etc/ftpsync': ensure => present, owner => 'mirror', group => 'mirror', @@ -62,6 +36,13 @@ content => template('mirrors/ftpsync.conf.erb'), } + file { '/var/log/ftpsync': + ensure => directory, + owner => 'mirror', + group => 'mirror', + mode => '0755', + } + # allow the Debian syncproxy to trigger ftpsync runs over ssh ssh::userkey { 'mirror': source => 'puppet:///modules/mirrors/ssh-debian-archvsync.pub', diff --git a/modules/mirrors/spec/classes/mirrors_debian_spec.rb b/modules/mirrors/spec/classes/mirrors_debian_spec.rb index 88d7159..ed01c24 100644 --- a/modules/mirrors/spec/classes/mirrors_debian_spec.rb +++ b/modules/mirrors/spec/classes/mirrors_debian_spec.rb @@ -10,16 +10,7 @@ }) end it do - should contain_file('/var/lib/mirror/archvsync/').with({ - 'ensure' => 'directory', - 'owner' => 'mirror', - 'group' => 'mirror', - 'mode' => '0755', - 'source' => 'puppet:///modules/mirrors/archvsync', - }) - end - it do - should contain_file('/var/lib/mirror/archvsync/etc/ftpsync.conf').with({ + should contain_file('/etc/ftpsync').with({ 'ensure' => 'present', 'owner' => 'mirror', 'group' => 'mirror', @@ -27,7 +18,11 @@ }) end it do - # 18eaf6c70 changed to push mirroring - should_not contain_cron('update-debian-mirror') + should contain_file('/var/log/ftpsync/').with({ + 'ensure' => 'directory', + 'owner' => 'mirror', + 'group' => 'mirror', + 'mode' => '0755', + }) end end diff --git a/modules/mirrors/templates/ftpsync.conf.erb b/modules/mirrors/templates/ftpsync.conf.erb index 322c8c2..844fe2a 100644 --- a/modules/mirrors/templates/ftpsync.conf.erb +++ b/modules/mirrors/templates/ftpsync.conf.erb @@ -7,5 +7,6 @@ MAILTO="root" ERRORSONLY="true" +LOGDIR="/var/log/ftpsync" FULLLOGS="false" LOGROTATE=14 -- To view, visit https://gerrit.wikimedia.org/r/384791 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I1cca0f806a014b5c05f7296b78eda7c2004c79a9 Gerrit-PatchSet: 2 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Faidon Liambotis <fai...@wikimedia.org> Gerrit-Reviewer: Faidon Liambotis <fai...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits