------------------------------------------------------------ revno: 260 committer: Sylvain Beucler <[email protected]> branch nick: infra timestamp: Sun 2011-02-13 15:59:24 +0100 message: Generate backup script from a simpler spec (but not from #file: tags in the docs) + check that files are correctly downloaded added: backup/base.chb backup/cherry-backup.py modified: backup/dl.sh
=== added file 'backup/base.chb' --- a/backup/base.chb 1970-01-01 00:00:00 +0000 +++ b/backup/base.chb 2011-02-13 14:59:24 +0000 @@ -0,0 +1,126 @@ +# These are the files necessary to rebuild the system. They come as a +# complement to the .txt instructions files from the 'administration' +# repository. Ideally we should automatically generate this file +# using the '#file:' annotations in those files. + +# Do not backup generatable or sed-able files. Document how to +# produce them instead. + +# We should make these files public so that people could easily +# reproduce the Savannah configuration. Backup confidential files +# (such as 'authorized_files') using 'dl-confidential.sh'. + [email protected]:/ -> colonialone.fsf.org/ + /root/.profile + /root/remote_backup.sh + + # blah + /home/syncaliases/00_aliases/aliases + /home/syncaliases/00_aliases/README + /home/syncaliases/.ssh/authorized_keys + + /etc/aliases + /etc/cron.daily/backup-bind + /etc/diffmon/diffmon.cf + + /etc/xen/xend-config.sxp + /etc/xen/auto/*** + /etc/xen/disabled/*** + + /etc/network/interfaces + /etc/network/firewall.sh + [email protected]:/ -> frontend.in.sv.gnu.org/ + /etc/cron.daily/sv_list_groups + /etc/cron.d/sv_export + + /etc/savane/.savane.conf.php + /etc/savane/savane.conf.pl + + /etc/apache2/sites-available/*** + /etc/apache2/conf.d/detect_bot.conf + [email protected]:/ -> vcs-noshell.in.sv.gnu.org/ + /etc/init.d/cvs-permissions + /etc/init.d/cvs_lockdirs + /etc/libnss-mysql.cfg + /etc/libnss-mysql-root.cfg + + /etc/cron.hourly/bzr_commit_mail_notification + /etc/cron.d/rsync_external_cvs_repositories + /etc/cron.d/sv + /etc/cron.d/cvs2git + /etc/cron.d/truncate-gitcvs-db-log + + /etc/apache2/sites-available/*** + /etc/apache2/conf.d/detect_bot.conf + /etc/apache2/conf.d/rlimit + /etc/apache2/conf.d/status + + --exclude /var/www/*/webalizer/* + /var/www/bzr/*** + /var/www/cvs/*** + /var/www/git/*** + /var/www/hg/*** + /var/www/svn/*** + /var/www/off-site/README + /var/www/off-site/hgweb/*** + /var/www/off-site/viewvc/viewvc.conf + /var/www/off-site/viewvc/templates/include/header.ezt + + /etc/gitweb.conf + /etc/cgitrc + /etc/mercurial/*** + +[shell] +# Mangle passwords (TODO: split them in separate file) +sed -i -e 's/^password.*/password XXXXX/' \ + vcs-noshell.in.sv.gnu.org/etc/libnss-mysql.cfg \ + vcs-noshell.in.sv.gnu.org/etc/libnss-mysql-root.cfg +# TODO: document hgweb/viewvc/cgit/etc. configurations instead of +# copying them. +[/shell] + [email protected]:/ -> sftp.in.sv.gnu.org/ + /etc/cron.d/download-tidyperms + /etc/cron.d/download-timestamp + /etc/cron.d/sv + + /etc/mirmon.conf + + /etc/apache2/sites-available/*** + /etc/apache2/conf.d/sv_dotsig + + --exclude /var/www/*/webalizer/* + --exclude /var/www/arch/google* + /var/www/arch/*** + /var/www/download/*** + /var/www/audio-video/*** + + /srv/download/00_MIRRORS.* + [email protected]:/ -> internal.in.sv.gnu.org/ + /usr/sbin/rmlist + /usr/sbin/config_list + /usr/lib/mailman/bin/change_pw + + /etc/mysql/my.cnf + /etc/mysql/my-huge.cnf + /etc/exim4/update-exim4.conf.conf + /etc/munin/munin.conf + + /etc/cron.d/sv + [email protected]:/ -> builder.in.sv.gnu.org/ + /usr/src/patched/README + /usr/src/patched/cvs-patches/*** + /usr/src/patched/webalizer.changelog + /usr/src/patched/debs/README + [email protected]:/ -> savannah-backup.gnu.org/ + /root/restore.sh + + /etc/cron.daily/check-savannah-backup + /etc/cron.d/savannah-backup + + /mnt/backup/backup.sh
=== added file 'backup/cherry-backup.py' --- a/backup/cherry-backup.py 1970-01-01 00:00:00 +0000 +++ b/backup/cherry-backup.py 2011-02-13 14:59:24 +0000 @@ -0,0 +1,180 @@ +#!/usr/bin/python + +# Cherry Backup +# Copyright (C) 2011 Sylvain Beucler +# +# Cherry Backup is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Cherry Backup is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +# Backup script to get the minimum set of files necessary to rebuild a +# system. We also check if all files are well rsync'd, so we can +# track typos and renamed files. + +# Do not backup generatable or sed-able files. Document how to +# produce them instead. + + +# Input file (base.chb) format: +""" +root@mybox:/ -> mybox-backup/ + /etc/aliases + /etc/mysql/my.cnf + /etc/exim4/update-exim4.conf.conf + /etc/munin/munin.conf + + /etc/apache2/sites-available/*** + + --exclude /var/www/*/webalizer/* + /var/www/site1/*** + /var/www/site2/*** + + /etc/libnss-mysql.cfg + /etc/libnss-mysql-root.cfg + +[shell] +# Mangle passwords +sed -i -e 's/^password.*/password XXXXX/' \ + mybox-backup/etc/libnss-mysql.cfg \ + mybox-backup/etc/libnss-mysql-root.cfg +[/shell] +""" + +import re +import os.path +import subprocess +import glob + +filename_in = 'base.chb' +fin = open(filename_in) +fout = open('dl.sh', 'w') +cur_dir = None +mode_shell_input = False +mode_files = False +print_separation_line = False +already_seen_paths = {} +files = {} + +fout.write("#!/bin/bash\n\n") +fout.write("# Generated by Cherry Backup - do not modify directly, edit '%s' instead\n" % filename_in) + +while True: + # "Parse" line-by-line + line = fin.readline() + if not line: + break + + # Shell throughput mode + if mode_shell_input: + if line.startswith('[/shell]'): + mode_shell_input = False + else: + fout.write(line) + # Config mode + else: + # comments + if line.strip().startswith('#'): + pass + elif line.strip() == '': + print_separation_line = True + # section + elif not line.startswith(' ') and not line.startswith('\t'): + # shell subsection + if line.startswith('[shell]'): + fout.write(" \\\n --exclude '*'\n") + mode_shell_input = True + mode_files = False + # host section + else: + match = re.match(r'^([^ ]+) -> ([^ ]+)\n', line) + if match: + src = match.groups()[0] + cur_dir = match.groups()[1] + if not cur_dir.endswith('/'): + cur_dir += '/' + if mode_files: + fout.write(" \\\n --exclude '*'\n") + mode_files = False + fout.write("\n") + fout.write("\n") + fout.write("echo '* %s -> %s'\n" % (src, cur_dir)) + fout.write("rsync -zavHS --delete-excluded %s %s \\\n" % (src, cur_dir)) + fout.write(" \\\n") + fout.write(" --exclude '*~' \\\n") + already_seen_paths = {} + files[cur_dir] = [] + print_separation_line = True + # Files: include or rsync option, with leading space + else: + mode_files = True + if print_separation_line: + fout.write(" \\\n") + print_separation_line = False + line = line.strip() + # rsync option + if line.startswith('-'): + fout.write(" " + line + " \\\n") + # include + else: + fullpath_elts = line.split('/') # /usr/src/a -> ['', 'usr', 'src', 'a'] + i = 0 + # Write each path directory + nb = len(fullpath_elts) + if fullpath_elts[-1] == '***': + nb -= 1 + for i in range(2, nb): + cur_path = '/'.join(fullpath_elts[0:i]) + if (i < len(fullpath_elts)): + cur_path += '/' + if not already_seen_paths.has_key(cur_path): + already_seen_paths[cur_path] = True + fout.write(" --include '%s' \\\n" % cur_path.replace("'", "'\\''")) + fout.write(" --include '%s' \\\n" % line.replace("'", "'\\''")) + files[cur_dir].append(line) +if mode_files: + fout.write(" \\\n --exclude '*'\n") + mode_files = False +fout.close() + + +# Run the generated script +subprocess.call(['bash', 'dl.sh']) + + +# Check if specified files were truly rsync'd +print "" +for host in files: + for bakpath in files[host]: + bakpath = re.sub('^/*', '', bakpath) + # There's a wildcard: + if bakpath.find('*') != -1 or bakpath.find('?') != -1: + bakdir = re.sub('\**$', '', bakpath) + if bakpath.endswith('*') and bakdir.endswith('/'): + if not os.path.isdir(os.path.join(host,bakdir)): + print "ERROR: " + os.path.join(host,bakdir) + " is not a directory" + elif not glob.glob(os.path.join(host,bakpath)): + print "ERROR: " + os.path.join(host,bakpath) + " has no match" + # TODO: we don't support several situations ('**' + # notation; trailing '*' in addition to other wildcard + # usage; ...?) + # Bare file or (empty) directory: + elif not os.path.exists(os.path.join(host,bakpath)): + print "ERROR: " + os.path.join(host,bakpath) + " is missing" + + +# TODO: If target directory is a SVN/Git/Bzr repository, prepare it +# for commit (deleted files marked as deleted, new files marked as +# added, etc.). Do not commit, but support being run several times +# (aka support files already marked for addition/suppression but +# non-committed, possibly with a simple 'git checkout' before the +# rsync). === modified file 'backup/dl.sh' --- a/backup/dl.sh 2010-12-21 19:18:00 +0000 +++ b/backup/dl.sh 2011-02-13 14:59:24 +0000 @@ -1,19 +1,10 @@ #!/bin/bash -# These are the files necessary to rebuild the system. They come as a -# complement to the .txt instructions files from the 'administration' -# repository. Ideally we should automatically generate this file -# using the '#file:' annotations in those files (we could also check -# if all files are well rsync'd, so we could track typos). - -# Do not backup generatable or sed-able files. Document how to -# produce them instead. - -# We should make these files public so that people could easily -# reproduce the Savannah configuration. Backup confidential files -# (such as 'authorized_files') using 'dl-confidential.sh'. - -rsync -avHS --delete-excluded [email protected]:/ colonialone.fsf.org/ \ +# Generated by Cherry Backup - do not modify directly, edit 'base.chb' instead + + +echo '* [email protected]:/ -> colonialone.fsf.org/' +rsync -zavHS --delete-excluded [email protected]:/ colonialone.fsf.org/ \ \ --exclude '*~' \ \ @@ -36,42 +27,43 @@ --include '/etc/diffmon/' \ --include '/etc/diffmon/diffmon.cf' \ \ - --include '/etc/' \ --include '/etc/xen/' \ --include '/etc/xen/xend-config.sxp' \ --include '/etc/xen/auto/***' \ --include '/etc/xen/disabled/***' \ \ - --include '/etc/' \ --include '/etc/network/' \ --include '/etc/network/interfaces' \ --include '/etc/network/firewall.sh' \ \ --exclude '*' -rsync -avHS --delete-excluded [email protected]:/ frontend.in.sv.gnu.org/ \ + +echo '* [email protected]:/ -> frontend.in.sv.gnu.org/' +rsync -zavHS --delete-excluded [email protected]:/ frontend.in.sv.gnu.org/ \ \ --exclude '*~' \ \ --include '/etc/' \ --include '/etc/cron.daily/' \ --include '/etc/cron.daily/sv_list_groups' \ + --include '/etc/cron.d/' \ --include '/etc/cron.d/sv_export' \ \ - --include '/etc/' \ --include '/etc/savane/' \ --include '/etc/savane/.savane.conf.php' \ --include '/etc/savane/savane.conf.pl' \ \ - --include '/etc/' \ --include '/etc/apache2/' \ - --include '/etc/apache2/sites-availables/***' \ + --include '/etc/apache2/sites-available/***' \ --include '/etc/apache2/conf.d/' \ --include '/etc/apache2/conf.d/detect_bot.conf' \ \ --exclude '*' -rsync -avHS --delete-excluded [email protected]:/ vcs-noshell.in.sv.gnu.org/ \ + +echo '* [email protected]:/ -> vcs-noshell.in.sv.gnu.org/' +rsync -zavHS --delete-excluded [email protected]:/ vcs-noshell.in.sv.gnu.org/ \ \ --exclude '*~' \ \ @@ -82,7 +74,6 @@ --include '/etc/libnss-mysql.cfg' \ --include '/etc/libnss-mysql-root.cfg' \ \ - --include '/etc/' \ --include '/etc/cron.hourly/' \ --include '/etc/cron.hourly/bzr_commit_mail_notification' \ --include '/etc/cron.d/' \ @@ -91,7 +82,6 @@ --include '/etc/cron.d/cvs2git' \ --include '/etc/cron.d/truncate-gitcvs-db-log' \ \ - --include '/etc/' \ --include '/etc/apache2/' \ --include '/etc/apache2/sites-available/***' \ --include '/etc/apache2/conf.d/' \ @@ -99,9 +89,9 @@ --include '/etc/apache2/conf.d/rlimit' \ --include '/etc/apache2/conf.d/status' \ \ + --exclude /var/www/*/webalizer/* \ --include '/var/' \ --include '/var/www/' \ - --exclude '/var/www/*/webalizer/*' \ --include '/var/www/bzr/***' \ --include '/var/www/cvs/***' \ --include '/var/www/git/***' \ @@ -116,7 +106,6 @@ --include '/var/www/off-site/viewvc/templates/include/' \ --include '/var/www/off-site/viewvc/templates/include/header.ezt' \ \ - --include '/etc/' \ --include '/etc/gitweb.conf' \ --include '/etc/cgitrc' \ --include '/etc/mercurial/***' \ @@ -130,7 +119,8 @@ # copying them. -rsync -avHS --delete-excluded [email protected]:/ sftp.in.sv.gnu.org/ \ +echo '* [email protected]:/ -> sftp.in.sv.gnu.org/' +rsync -zavHS --delete-excluded [email protected]:/ sftp.in.sv.gnu.org/ \ \ --exclude '*~' \ \ @@ -140,19 +130,17 @@ --include '/etc/cron.d/download-timestamp' \ --include '/etc/cron.d/sv' \ \ - --include '/etc/' \ --include '/etc/mirmon.conf' \ \ - --include '/etc/' \ --include '/etc/apache2/' \ --include '/etc/apache2/sites-available/***' \ --include '/etc/apache2/conf.d/' \ --include '/etc/apache2/conf.d/sv_dotsig' \ \ + --exclude /var/www/*/webalizer/* \ + --exclude /var/www/arch/google* \ --include '/var/' \ --include '/var/www/' \ - --exclude '/var/www/*/webalizer/*' \ - --exclude '/var/www/arch/google*' \ --include '/var/www/arch/***' \ --include '/var/www/download/***' \ --include '/var/www/audio-video/***' \ @@ -163,7 +151,9 @@ \ --exclude '*' -rsync -avHS --delete-excluded [email protected]:/ internal.in.sv.gnu.org/ \ + +echo '* [email protected]:/ -> internal.in.sv.gnu.org/' +rsync -zavHS --delete-excluded [email protected]:/ internal.in.sv.gnu.org/ \ \ --exclude '*~' \ \ @@ -171,7 +161,6 @@ --include '/usr/sbin/' \ --include '/usr/sbin/rmlist' \ --include '/usr/sbin/config_list' \ - --include '/usr/' \ --include '/usr/lib/' \ --include '/usr/lib/mailman/' \ --include '/usr/lib/mailman/bin/' \ @@ -180,17 +169,20 @@ --include '/etc/' \ --include '/etc/mysql/' \ --include '/etc/mysql/my.cnf' \ + --include '/etc/mysql/my-huge.cnf' \ + --include '/etc/exim4/' \ --include '/etc/exim4/update-exim4.conf.conf' \ + --include '/etc/munin/' \ --include '/etc/munin/munin.conf' \ \ - --include '/etc/' \ --include '/etc/cron.d/' \ --include '/etc/cron.d/sv' \ \ --exclude '*' -rsync -avHS --delete-excluded [email protected]:/ builder.in.sv.gnu.org/ \ +echo '* [email protected]:/ -> builder.in.sv.gnu.org/' +rsync -zavHS --delete-excluded [email protected]:/ builder.in.sv.gnu.org/ \ \ --exclude '*~' \ \ @@ -205,13 +197,17 @@ \ --exclude '*' -rsync -avHS [email protected]:/ savannah-backup.gnu.org/ \ + +echo '* [email protected]:/ -> savannah-backup.gnu.org/' +rsync -zavHS --delete-excluded [email protected]:/ savannah-backup.gnu.org/ \ + \ + --exclude '*~' \ \ --include '/root/' \ --include '/root/restore.sh' \ \ --include '/etc/' \ - --include '/etc/cron.daily' \ + --include '/etc/cron.daily/' \ --include '/etc/cron.daily/check-savannah-backup' \ --include '/etc/cron.d/' \ --include '/etc/cron.d/savannah-backup' \
_______________________________________________ Savannah-cvs mailing list [email protected] http://lists.gnu.org/mailman/listinfo/savannah-cvs
