#!/bin/bash
#
# Copyright (c) 2008-2009 Tino Schwarze Community4you GmbH <tino.schwarze@community4you.de>
#
# Purpose
# =======
#
# This script may be called by Bacula (or your custom backup solution)
# before server backups should happen.
# It collects recently backed up BackupPC hosts and creates
# TAR.BZ2 files from their most recent backup.
#
# If it is given the parameter "afterJob", it will remove all files from
# the totape directory.
#
# You may exclude a host from automated tape-saving by creating a file
# prepareDumpManually in the appropiate pc/ subdirectory.
#
# License
# =======
# 
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
# 
# You should have received a copy of the GNU General Public License in the
# LICENSE file along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.


set -a

# for nicer date formats
LANG=en_GB.utf8

BACKUPPCDIR=/backup/backuppc
# where to store the TAR files
ARCHIVEDIR=/data/bacula-scratch/backuppc-archive/
# how many percent of the archive file system need to stay free
ARCHIVEFSMINFREE=10
# at which free space percentage to delete created archives
ARCHIVEFSFREEABORT=1
# upper limit to store to tape (in megabytes)
MAXSIZETOTAPE=300000
MAXOVERALLSIZE=50000
# maximum time for this script to run (in seconds)
# 10h: 36000, 12h: 43200, 18h: 64800, 20h: 72000
MAXRUNTIME=36000
# do not start another job if less than this seconds runtime left
MINJOBRUNTIME=3600
set +a

#### No configuration below this line #####
# 

prepareddir="$ARCHIVEDIR/prepared/"
manualdir="$ARCHIVEDIR/manual/"
totapedir="$ARCHIVEDIR/totape/"

[ $MINJOBRUNTIME -gt $MAXRUNTIME ] && MINJOBRUNTIME=$MAXRUNTIME

function die ()
{
	echo "$*" >&2
	exit 1
}

if [ "$1" = "afterJob" ] ; then

	cd  "$totapedir" || die "could not cd to $totapedir"

	for archivefile in *.tar.bz2 ; do
		[ -f "$archivefile" ] || continue
		# determine hostname from archive file name
		host="$(echo "$archivefile" | sed -e 's/\(.*\)\.[0-9]\+\.tar\.bz2/\1/' )"
		if [ -d "$BACKUPPCDIR/pc/$host" ] ; then
			ls -l "$archivefile" >> "$BACKUPPCDIR/pc/$host/dumpsOnTape"
			echo "removing $archivefile"
			rm -f "$archivefile" || echo "WARNING: could not remove $totapedir/$archivefile"
		else
			echo "Could not find host $host for archive $archivefile"
			continue
		fi
	done
	exit 0
fi

# get day of week (1=monday, 7=sunday)
weekday="$(date +%u)"

# process BackupPC hosts file, create list of hosts
hostlist="$(grep -v '^#\|^ *$' "$BACKUPPCDIR/conf/hosts" | awk '/^host/{next} {print $1}')"

# build list of hosts to backup (e.g. never archived or newer backup)
hosts2backuplist=""
for h in $hostlist ; do
	dumpsfile="$BACKUPPCDIR/pc/$h/dumpsPrepared"
	backupsfile="$BACKUPPCDIR/pc/$h/backups"

	# skip hosts without backups
	[ -f "$backupsfile" ] || continue

	# special handling for several hosts: do dump preparation manually (or via cron)
	[ -f "$BACKUPPCDIR/pc/$h/prepareDumpManually" ] && continue

	# skip hosts with currently running backups
	[ ! -e "$BACKUPPCDIR/pc/$h/new" ] || continue

	if [ ! -f "$dumpsfile" ] ; then
		hosts2backuplist="$hosts2backuplist $dumpsfile"
		touch --reference="$BACKUPPCDIR/pc/$h/backups" $dumpsfile
	elif [ ! "$dumpsfile" -nt "$backupsfile" ] ; then
		hosts2backuplist="$hosts2backuplist $dumpsfile"
	elif [ $weekday -eq 7 ] ; then
		# on Sunday, check for very long not backed up hosts
		# (we keep some hosts, so we'd rather Tape backup them too from time to time)
		# TODO
		echo "TODO: check for long-time not backed up hosts on Sundays"
		# max. number of obsolete backups per sunday: 1
	fi
done

# now sort hosts: prefer hosts which have not been backed up for a long time
hosts2backuplist="$(ls -tr $hosts2backuplist)"

echo $hosts2backuplist

# Thanks to Dave who posted this function as a comment in
# http://cateee.blogspot.com/2006/10/shell-parallel-execution-with-timeouts.html
# :-)
function runWithTimeout ()
{
    if [ $# -lt 2 ]; then
        echo "usage: $0 <timeout s> <commandline>"
        return 1
    fi
    timeout=$1
    shift

    # Start the command.
    "$@" &
    cmdPid=$!

    # Start the timeout process.
    ( sleep $timeout; echo "$1 timed out (pid $cmdPid)"; kill -ALRM $cmdPid ) &
    sleepPid=$!

    # Don't leak the children if we're interrupted by ^C etc.
    trap "kill $cmdPid $sleepPid; exit 5" INT TERM EXIT

    # Block until the command exits or times out.
    wait $cmdPid
    cmdStatus=$?

    # Clear out the ^C trap.
    trap - INT TERM EXIT

    if [ $cmdStatus -le 128 ] ; then
        # The command exited on its own; kill the timeout process.
        # This will kill the background shell, not the sleep
        # process started by it, so the sleep itself may hang around
        # for a while. But, since the subshell will be gone when it
        # exits, nothing will try to kill $cmdPid.
        kill $sleepPid
    fi
    # The caller can check the return value against 128 to see
    # if the command timed out or simply failed.
    return $cmdStatus
}

# creates an archive
# arguments:
# $1 = hostname
# $2 = backupno
# $3 = name of archive to create
function createArchive ()
{
	su backuppc -c "$BACKUPPCDIR/BackupPC/bin/BackupPC_tarCreate -t -h $1 -n $2 -s '*' ." | bzip2 > "$3" &
	#su backuppc -c "$BACKUPPCDIR/BackupPC/bin/BackupPC_tarCreate -t -h $1 -n $2 -s '*' ." | ( dd bs=1M count=5 ; exit 1 ; ) > "$3" &
	cmdPid="$!"
	# kill su child as well if we get killed by timeout
	trap "kill $cmdPid" ALRM TERM EXIT
	wait $cmdPid
	cmdStatus="$?"
	trap - ALRM TERM EXIT
	return $cmdStatus
}

#runWithTimeout 60 createArchive herkules 370 "$prepareddir/herkules.370.tar.bz2"
#echo "command exited with state $?"
#exit 1

preparedcounter=0
overallsize=0
overalltapesize=0
starttime=$(date +%s)

for dumpsfile in $hosts2backuplist ; do

	# check whether max runtime is over
	[ $(($(date +%s)-$starttime)) -ge $MAXRUNTIME ] && break

	hostdir="${dumpsfile%/dumpsPrepared}"
	# extract host name from path to dumpsfile
	host="${hostdir#$BACKUPPCDIR/pc/}"

	# for now, skip zentaur: it takes too long to archive
	[ "$host" = "zentaur" ] && continue

	# get number of latest backup of host
	latestbackupno="$(tail -1 "$hostdir/backups" | awk '{print $1}')"

	# no backup, yet?
	[ -z "$latestbackupno" ] && continue

	archivefilename="${host}.${latestbackupno}.tar.bz2"
	preparedarchive="$prepareddir/$archivefilename"
	totapearchive="$totapedir/$archivefilename"

	# create archive if it doesn't exist already
	if [ -f "$totapearchive" ] ; then
		# totape archive already exists - reuse it
		mv "$totapearchive" "$preparedarchive"
	elif [ ! -f "$preparedarchive" ] ; then
		# remove obsolete archives
		rm -f "$prepareddir/${host}.*"
		rm -f "$totapedir/${host}.*"

		# check if there is enough time left for another job
		timeleft="$(($starttime+$MAXRUNTIME-$(date +%s)))"
		[ $timeleft -lt $MINJOBRUNTIME ] && { echo "only $timeleft seconds left for job - aborting" ; break ; }

		# check free space on archive FS
		freeblockspercent="$(($(stat '--format=100*%a/%b' -f $ARCHIVEDIR)))"
		# do not create new archive (but continue, so old archives are probably removed and small ones created)
		[ $freeblockspercent -lt $ARCHIVEFSMINFREE ] && continue

		# create new archive
		echo "creating new archive $preparedarchive (timeout: $timeleft seconds)..."

		let preparedcounter=preparedcounter+1

		runWithTimeout $timeleft createArchive $host $latestbackupno "$preparedarchive"
		res=$?
		if [ $res -ge 128 ] ; then
			echo "WARNING: archiving timed out or failed (exit code: $res)."
			rm -f "$preparedarchive"
			if [ $preparedcounter -eq 1 ] ; then
				echo "SEVERE: The archive could not be prepared within total timeout. Postponing archive creation for a long time."
				echo "`date +%Y%m%d`: archive creation timed out" >> "$hostdir/dumpsPrepared" 
			fi
			continue
		elif [ $res -ne 0 ] ; then
			echo "WARNING: archiving failed (exit code: $res)."
			rm -f "$preparedarchive"
			continue
		fi

		# check free space on archive FS
		freeblockspercent="$(($(stat '--format=100*%a/%b' -f $ARCHIVEDIR)))"
		if [ $freeblockspercent -lt $ARCHIVEFSFREEABORT ] ; then
			echo "removing generated archive $preparedarchive because archive disk got too full"
			rm "$preparedarchive"
			continue
		fi
	fi

	# now check created archive's size (in megabytes)
	archivesize="$(($(stat --format=%s "$preparedarchive")/1024/1024))"
	
	if [ $archivesize -gt $MAXSIZETOTAPE ] ; then
		echo "CRITICAL: The archive will never fit on a tape. (archive size: $archivesize MB, allowed tape size: $MAXSIZETOTAPE MB)"
	fi

	let overallsize=overallsize+archivesize

	# check whether the new archive still fits to tape
	if [ $((overalltapesize+archivesize)) -lt $MAXSIZETOTAPE ] ; then
		# write log of dumps on tape
		ls -l "$preparedarchive" >> "$hostdir/dumpsPrepared" 
		# move created archive totape
		mv "$preparedarchive" "$totapedir"
		let overalltapesize=overalltapesize+archivesize
	fi

done

# now add manually prepared archives (on Sunday only)
if [ $weekday -eq 7 ] ; then
	let manualsize=0

	for preparedarchive in $manualdir/*.tar.* ; do
		[ -f "$preparedarchive" ] || continue

		archivesize="$(($(stat --format=%s "$preparedarchive")/1024/1024))"

		mv "$preparedarchive" "$totapedir"

		let manualsize=manualsize+archivesize
		let overallsize=overallsize+archivesize
	done
fi

echo "$overalltapesize megabytes prepared for tape archive, backing up $overallsize megabytes in archives total, $(($(date +%s)-$starttime)) seconds elapsed time"