Hello community, here is the log from the commit of package transactional-update for openSUSE:Factory checked in at 2020-09-24 16:13:27 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/transactional-update (Old) and /work/SRC/openSUSE:Factory/.transactional-update.new.4249 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "transactional-update" Thu Sep 24 16:13:27 2020 rev:59 rq:835888 version:2.25.1 Changes: -------- --- /work/SRC/openSUSE:Factory/transactional-update/transactional-update.changes 2020-08-13 10:17:08.574732028 +0200 +++ /work/SRC/openSUSE:Factory/.transactional-update.new.4249/transactional-update.changes 2020-09-24 16:14:04.572888547 +0200 @@ -1,0 +2,20 @@ +Mon Sep 21 16:17:12 UTC 2020 - Ignaz Forster <ifors...@suse.com> + +- Version 2.25.1 + - Fix inotify watcher setup + - Use log_{info,error} for more messages to avoid messing up Salt logs + +------------------------------------------------------------------- +Mon Sep 21 14:09:41 UTC 2020 - Ignaz Forster <ifors...@suse.com> + +- Version 2.25 + - Reduce number of overlays: + Instead of using transparent overlays for all previous layers only add the + previous snapshot's overlay; this will greatly reduce the number of + overlays while still making sure that /etc changes in the running system + will be visible in the next snapshot + - When using --drop-if-no-change synchronize potential /etc changes with + running system + - Exclude all non-root-fs mounts from inotify watcher + +------------------------------------------------------------------- Old: ---- transactional-update-2.24.1.tar.gz New: ---- transactional-update-2.25.1.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ transactional-update.spec ++++++ --- /var/tmp/diff_new_pack.ZcasC1/_old 2020-09-24 16:14:05.996889972 +0200 +++ /var/tmp/diff_new_pack.ZcasC1/_new 2020-09-24 16:14:06.000889975 +0200 @@ -17,7 +17,7 @@ Name: transactional-update -Version: 2.24.1 +Version: 2.25.1 Release: 0 Summary: Transactional Updates with btrfs and snapshots License: GPL-2.0-or-later ++++++ transactional-update-2.24.1.tar.gz -> transactional-update-2.25.1.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/transactional-update-2.24.1/NEWS new/transactional-update-2.25.1/NEWS --- old/transactional-update-2.24.1/NEWS 2020-08-12 12:04:41.000000000 +0200 +++ new/transactional-update-2.25.1/NEWS 2020-09-21 18:16:14.000000000 +0200 @@ -2,6 +2,20 @@ Copyright (C) 2016-2019 Thorsten Kukuk et al. +Version 2.25.1 +* Fix inotify watcher setup +* Use log_{info,error} for more messages to avoid messing up Salt logs + +Version 2.25 +* Reduce number of overlays: + Instead of using transparent overlays for all previous layers only add the + previous snapshot's overlay; this will greatly reduce the number of overlays + while still making sure that /etc changes in the running system will be + visible in the next snapshot +* When using --drop-if-no-change synchronize potential /etc changes with + running system +* Exclude all non-root-fs mounts from inotify watcher + Version 2.24.1 * SELinux: adjust labels for etc, fstab and grub.cfg diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/transactional-update-2.24.1/configure.ac new/transactional-update-2.25.1/configure.ac --- old/transactional-update-2.24.1/configure.ac 2020-08-12 12:04:41.000000000 +0200 +++ new/transactional-update-2.25.1/configure.ac 2020-09-21 18:16:14.000000000 +0200 @@ -1,5 +1,5 @@ dnl Process this file with autoconf to produce a configure script. -AC_INIT(transactional-update, 2.24.1) +AC_INIT(transactional-update, 2.25.1) AM_INIT_AUTOMAKE AC_PREFIX_DEFAULT(/usr) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/transactional-update-2.24.1/doc/transactional-update.xml new/transactional-update-2.25.1/doc/transactional-update.xml --- old/transactional-update-2.24.1/doc/transactional-update.xml 2020-08-12 12:04:41.000000000 +0200 +++ new/transactional-update-2.25.1/doc/transactional-update.xml 2020-09-21 18:16:14.000000000 +0200 @@ -446,21 +446,36 @@ <filename class='directory'>/var/lib/overlay/<snapshotnum>/etc</filename>. </para> <para> - Each snapshot will have one associated overlay directory. The overlay - directories are mounted using overlay stacking, i.e. the current - snapshot will mount its associated overlay as the - <option>upperdir</option> and the previous snapshot's overlays as - <option>lowerdir</option>. That means that the current snapshot will - see all the changes accumulated in the previous snapshots, but older - snapshots will not see changes applied to newer snapshots. + Each snapshot will have one associated overlay directory. On creating + a new snapshot the previous snapshot's + <filename class='directory'>/etc</filename> state will be sychronized + into the new snapshot and used as a base. The overlay directories of + the current and the new snapshot are then mounted using overlay + stacking, i.e. the new snapshot's overlay will be mounted as the + <option>upperdir</option> and the current snapshot's overlay as + <option>lowerdir</option>. This way changes applied to + <filename class='directory'>/etc</filename> after the snapshot was + taken, but before the reboot takes place, will still be visible to the + new snapshot (Exception: If the file has been modified both in the + current and the new snapshot, then the file state of the new snapshot + will be visible). + </para> + <para> + If the <option>--continue</option> is used multiple times to extend a + new snapshot while the system has not been rebooted, and if that + snapshot is based on the currently active system, then the + synchronization will only run for the first snapshot; the additional + snapshot layers will be added to <option>lowerdir</option>. Again + this is to make sure that changes to the running system will still be + visible after booting into the new system. </para> <para> Let's have a look at an example <filename>fstab</filename> entry: - <programlisting>overlay /etc overlay defaults,upperdir=/sysroot/var/lib/overlay/83/etc,lowerdir=/sysroot/var/lib/overlay/81/etc:/sysroot/var/lib/overlay/80/etc:/sysroot/var/lib/overlay/77/etc:/sysroot/var/lib/overlay/76/etc:/sysroot/etc,workdir=/sysroot/var/lib/overlay/work-etc,x-systemd.requires-mounts-for=/var,x-systemd.requires-mounts-for=/var/lib/overlay,x-systemd.requires-mounts-for=/sysroot/var,x-systemd.requires-mounts-for=/sysroot/var/lib/overlay,x-initrd.mount 0 0</programlisting> + <programlisting>overlay /etc overlay defaults,upperdir=/sysroot/var/lib/overlay/82/etc,lowerdir=/sysroot/var/lib/overlay/81/etc:/sysroot/var/lib/overlay/76/etc:/sysroot/etc,workdir=/sysroot/var/lib/overlay/work-etc,x-systemd.requires-mounts-for=/var,x-systemd.requires-mounts-for=/var/lib/overlay,x-systemd.requires-mounts-for=/sysroot/var,x-systemd.requires-mounts-for=/sysroot/var/lib/overlay,x-initrd.mount 0 0</programlisting> </para> <itemizedlist> <listitem><para> - We are currently in snapshot <emphasis>83</emphasis> as indicated by the + We are currently in snapshot <emphasis>82</emphasis> as indicated by the <option>upperdir</option> directory. This can be confirmed by typing <command>snapper list</command> or <command>btrfs subvolume get-default /</command>. All changes to @@ -468,12 +483,18 @@ directory. </para></listitem> <listitem><para> - <option>lowerdir</option> contains several overlays from previous - snapshots. In this case it seem the snapshots were not created - consecutively - maybe there were a few rollbacks in between or a - snapshot wasn't applied by rebooting into it. The lowest snapshot is - always <filename class='directory'>/sysroot/etc</filename>, containing - the root file system's contents. + <option>lowerdir</option> contains two numbered overlay directories. + The later directory with number <emphasis>76</emphasis> indicates the + snapshot which was used as a base. This snapshot's + <filename class='directory'>/etc</filename> state was also synchronized + into the read-only root file system of snapshot 82. As the + <option>lowerdir</option> contains a second entry with number + <emphasis>81</emphasis> it means that the <option>--continue</option> + has been used before the system was rebooted. Gaps in the number such + as seen here may indicate that those snapshots were discarded or a + rollback to snapshot 76 was performed. The lowest snapshot is always + <filename class='directory'>/sysroot/etc</filename>, containing the + root file system's contents. </para></listitem> <listitem><para> As <filename class='directory'>/etc</filename> is mounted by @@ -484,16 +505,8 @@ </para></listitem> </itemizedlist> <para> - To prevent the <filename class='directory'>/etc</filename> overlays - from growing infinitely they will be synced into the root file system if - no snapshot is referencing them any more. So please be aware that the - root file system's <filename class='directory'>/etc</filename> state by - itself does neither contain the original package contents nor the - snapshot's current state. Only the snapshot plus all overlay layers - represents the state of <filename class='directory'>/etc</filename> - correctly. Overlays no longer needed because they have been merged into - all snapshots will be deleted during the <command>transactional-update - cleanup</command> run. + Overlays no longer referenced by any snapshots will be deleted during + the <command>transactional-update cleanup</command> run. </para> </section> <!-- diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/transactional-update-2.24.1/man/transactional-update.8.xml new/transactional-update-2.25.1/man/transactional-update.8.xml --- old/transactional-update-2.24.1/man/transactional-update.8.xml 2020-08-12 12:04:41.000000000 +0200 +++ new/transactional-update-2.25.1/man/transactional-update.8.xml 2020-09-21 18:16:14.000000000 +0200 @@ -104,11 +104,11 @@ <filename class='directory'>/var/lib/overlay</filename>. As configuration files may be modified after a snapshot was created and before a reboot is performed (e.g. via configuration management software) the overlay file system will use -multiple lower layers, i.e. changes in older layers will be visible in newer -snapshots, but not vice versa. To keep the number of layers at a minimum and the -amount of data stored in the overlays low the -<filename class='directory'>/etc</filename> state of the oldest still available -snapshot is synchronized into the new root file system; unused overlays will be +multiple lower layers, i.e. configuration file changes applied to the currently +running system will be visible in the new system, but not vice versa. To keep +the number of layers at a minimum the +<filename class='directory'>/etc</filename> state of the previous snapshot is +synchronized into the new root file system; unused overlays will be removed at a later time (see the <option>cleanup</option> option). </para> <warning> diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/transactional-update-2.24.1/sbin/transactional-update.in new/transactional-update-2.25.1/sbin/transactional-update.in --- old/transactional-update-2.24.1/sbin/transactional-update.in 2020-08-12 12:04:41.000000000 +0200 +++ new/transactional-update-2.25.1/sbin/transactional-update.in 2020-09-21 18:16:14.000000000 +0200 @@ -82,6 +82,7 @@ SNAPPER_NO_DBUS="" TMPFILE="" ETC_OVERLAY_DIR="" +ETC_OVERLAY_DIR_PREV="" ETC_OVERLAY_WORK_DIR="" INOTIFY_PID="" @@ -170,14 +171,14 @@ log_info() { echo `date "+%Y-%m-%d %H:%M"` "$@" >> ${LOGFILE} if [ ${VERBOSITY} -ge 2 ]; then - echo "$@" + echo -e "$@" fi } log_error() { TELEM_PAYLOAD="${TELEM_PAYLOAD}\nmessage=$@" echo `date "+%Y-%m-%d %H:%M"` "$@" >> ${LOGFILE} - echo "$@" 1>&4 + echo -e "$@" 1>&4 } bashlock() { @@ -264,11 +265,11 @@ done if [ -n "${SNAPSHOT_ID}" ] ; then - log_error "Removing snapshot #${SNAPSHOT_ID}..." + log_info "Removing snapshot #${SNAPSHOT_ID}..." snapper ${SNAPPER_NO_DBUS} delete ${SNAPSHOT_ID} |& tee -a ${LOGFILE} fi if [ -n "${ETC_OVERLAY_DIR}" ] ; then - log_error "Removing overlay directory ${ETC_OVERLAY_DIR%/*}..." + log_info "Removing overlay directory ${ETC_OVERLAY_DIR%/*}..." if findmnt "${ETC_OVERLAY_DIR}" >/dev/null; then umount "${ETC_OVERLAY_DIR}" fi @@ -290,6 +291,13 @@ kill ${INOTIFY_PID} log_info "The snapshot does not contain any changed files." INOTIFY_PID="" + + # Even if the snapshot itself did not contain any changes, + # /etc may do so. Changes in /etc may be applied immediately, + # so merge them back into the running system. + log_info "Merging overlay directory ${SNAPSHOT_DIR}/etc into /etc..." + rsync --archive --inplace --xattrs --acls --exclude 'fstab' --delete --quiet "${SNAPSHOT_DIR}/etc/" /etc + quit 0 fi @@ -431,10 +439,9 @@ local option lowerdir overlay_id fstab="" - fstab_active_lowerdirs=() fstab_lowerdirs=() fstab_upper="" - fstab_oldest_active_snapshot="" + fstab_previous_snapshot="" fstab_otheroptions="" # Check for /etc overlay @@ -455,18 +462,8 @@ for lowerdir in `echo ${option#*=} | tr ':' ' '`; do overlay_id=`echo "${lowerdir#/sysroot}" | sed -n 's#^/var/lib/overlay/\([[:digit:]]\+\)/etc$#\1#p'` - # Special handling for old overlay location - if [ "${lowerdir}" = "/sysroot/var/lib/overlay/etc" ]; then - if [ "${fstab_lowerdirs[*]}" = "${fstab_active_lowerdirs[*]}" ]; then - fstab_active_lowerdirs+=("${lowerdir}") - fi - # Add common directories and directories of existing snapshots - elif [ -z "${overlay_id}" -o -e "/.snapshots/${overlay_id}/snapshot" ]; then - if [ -n "${overlay_id}" ]; then - fstab_oldest_active_snapshot="/.snapshots/${overlay_id}/snapshot" - fstab_active_lowerdirs=("${fstab_lowerdirs[@]}") - fi - fstab_active_lowerdirs+=("${lowerdir}") + if [ -z "${fstab_previous_snapshot}" -a -n "${overlay_id}" ]; then + fstab_previous_snapshot="/.snapshots/${overlay_id}/snapshot" fi fstab_lowerdirs+=("${lowerdir}") done @@ -480,7 +477,6 @@ ;; esac done - fstab_oldest_active_snapshot="${fstab_oldest_active_snapshot:-/.snapshots/${CURRENT_SNAPSHOT_ID}/snapshot}" } # Replace all /sysroot prefixes from /etc fstab entry; for the overlays in /var @@ -502,16 +498,11 @@ if [ ${#fstab_lowerdirs[@]} -le 1 ]; then return fi - log_info "Syncing /etc of oldest snapshot $1 as base into new snapshot $2" + log_info "Syncing /etc of previous snapshot $1 as base into new snapshot $2" mount_opts="defaults," mount_opts+="lowerdir=" - # Special case: If the current in-use snapshot is the oldest snapshot - # already it will be added as a lowerdir, so don't mount it for - # syncing (it may change at any time anyway). - if [ "${fstab_upper}" != "${current_upper}" ]; then - mount_opts+="${fstab_upper#/sysroot}:" - fi + mount_opts+="${fstab_upper#/sysroot}:" mount_opts+="`parse_lowerdirs_for_mount $1`" mount -t overlay overlay "${ETC_OVERLAY_DIR}" -o ${mount_opts} if [ $? -ne 0 ]; then @@ -519,7 +510,7 @@ quit 1 fi - rsync --archive --inplace --xattrs --filter='-x security.selinux' --acls --delete --quiet "${ETC_OVERLAY_DIR}" "$2" + rsync --archive --inplace --xattrs --acls --delete --quiet "${ETC_OVERLAY_DIR}" "$2" if [ $? -ne 0 ]; then log_error "ERROR: syncing $1 into snapshot $2 failed!" quit 1 @@ -527,6 +518,17 @@ umount "${ETC_OVERLAY_DIR}" } +# Return the optimized list of mount points +root_mount_points() { + local prev=0 + for mount in $(findmnt --noheadings --submounts --target / --output TARGET --raw | sort | tail -n +2); do + if [ "${prev}/" != "${mount:0:$((${#prev}+1))}" ]; then + echo "${mount}" + prev="${mount}" + fi + done +} + ORIG_ARGS=("$@") while [ 1 ]; do @@ -734,7 +736,7 @@ if [ -z "${TA_UPDATE_TMPFILE}" ]; then bashlock if [ $? -ne 0 ]; then - echo "Couldn't get lock, is another instance already running?" + log_error "Couldn't get lock, is another instance already running?" exit 1 fi self_update "${ORIG_ARGS[@]}" @@ -757,7 +759,7 @@ SNAPPER_VERSION=`snapper --version | head -1 | cut -d ' ' -f 2` if [ -n "${BASE_SNAPSHOT_ID}" -a $(zypper --terse versioncmp $SNAPPER_VERSION 0.8.4) -lt 0 ]; then - echo "ERROR: snapper >= 0.8.4 required for --continue option!" + log_error "ERROR: snapper >= 0.8.4 required for --continue option!" log_info "transactional-update finished" telem_finish 1 exit 1 @@ -856,7 +858,7 @@ NEED_REBOOT_WARNING=1 fi if [ ${NEED_REBOOT_WARNING} -eq 1 ]; then - echo "Please reboot to finish rollback!" + log_error "Please reboot to finish rollback!" fi telem_finish 0 exit 0 @@ -998,11 +1000,15 @@ get_etc_overlay_from / current_upper="${fstab_upper}" - - # Copy the contents of oldest /etc layer into snapshot - sync_etc "${fstab_oldest_active_snapshot}" "${SNAPSHOT_DIR}" get_etc_overlay_from "/.snapshots/${BASE_SNAPSHOT_ID}/snapshot" + # Copy the previous snapshot's /etc state into new snapshot (if it + # hasn't been synced in a previous layer already) + if [ -n "${fstab_previous_snapshot}" ] && [[ ${fstab_lowerdirs[@]} != *"${current_upper}"* ]]; then + sync_etc "${fstab_previous_snapshot}" "${SNAPSHOT_DIR}" + get_etc_overlay_from "/.snapshots/${BASE_SNAPSHOT_ID}/snapshot" + fi + # Special handling /etc/fstab{.sys}: dracut needs the *current* version # of the files in the snapshot, as the overlay is not yet mounted # during early boot - in fact the files are required to be able to @@ -1025,10 +1031,19 @@ gawk -i inplace '$3 == "overlay" && $4 ~ "=/sysroot/var/lib/overlay/" { $4 = $4",x-systemd.requires-mounts-for=/var,x-systemd.requires-mounts-for=/var/lib/overlay,x-systemd.requires-mounts-for=/sysroot/var,x-systemd.requires-mounts-for=/sysroot/var/lib/overlay,x-initrd.mount"; } 1' "${SNAPSHOT_DIR}/etc/fstab" fi + # Check whether the new snapshot is based on the currently running + # one - if so, then keep the overlay stack to not loose changes + # applied until reboot + if [[ ${fstab_lowerdirs[@]} == *"${current_upper}"* ]]; then + lowerdirs="${fstab_upper}:$(echo ${fstab_lowerdirs[@]} | tr ' ' ':')" + else + lowerdirs="${fstab_upper}:/sysroot/etc" + fi + # Update fstab{,.sys} entries for /etc overlays fstab_line="overlay /etc overlay ${fstab_otheroptions}" fstab_line+="upperdir=/sysroot${ETC_OVERLAY_DIR}," - fstab_line+="lowerdir=${fstab_upper}:`echo ${fstab_active_lowerdirs[@]} | tr ' ' ':'`," + fstab_line+="lowerdir=${lowerdirs}," fstab_line+="workdir=/sysroot/var/lib/overlay/work-etc," fstab_line+="x-systemd.requires-mounts-for=/var," fstab_line+="x-systemd.requires-mounts-for=/var/lib/overlay," @@ -1097,11 +1112,16 @@ if [ ${ETC_IS_OVERLAY} -eq 1 ]; then lowerdir="${fstab_upper///sysroot\/var//var}:" - lowerdir+="`parse_lowerdirs_for_mount "/.snapshots/${BASE_SNAPSHOT_ID}/snapshot"`" - # Check whether the current upper directory is part of the snapshot's lower - # directory stack; if so use reuse /etc directly instead, as mounting - # the same upper directory multiple times is not supported by overlayfs - lowerdir="${lowerdir/${current_upper#/sysroot}*/\/etc}" + # Is the snapshot based on the currently running system? + if [[ ${fstab_lowerdirs[@]} == *"${current_upper}"* ]]; then + lowerdir+="$(parse_lowerdirs_for_mount "/.snapshots/${BASE_SNAPSHOT_ID}/snapshot")" + # Mounting an already used overlay directory again is not + # supported by overlayfs - use the old mount point instead + lowerdir="${lowerdir/${current_upper#/sysroot}*/\/etc}" + else + # Use the (synced) /etc directory of the new snapshot as base + lowerdir+="/.snapshots/${SNAPSHOT_ID}/snapshot/etc" + fi etc_opts="defaults," etc_opts+="lowerdir=${lowerdir}," @@ -1171,8 +1191,8 @@ if [ $DROP_IF_NO_CHANGE -eq 1 ]; then #TODO: Handle directories with special characters (space, newline, apostrophe etc) - for snapmount in `findmnt --noheadings --submounts --target "${SNAPSHOT_DIR}" --output TARGET --raw | grep "${SNAPSHOT_DIR}"`; do - INOTIFY_EXCLUDES="${INOTIFY_EXCLUDES} @${snapmount}" + for snapmount in $(root_mount_points); do + INOTIFY_EXCLUDES="${INOTIFY_EXCLUDES} @${SNAPSHOT_DIR}${snapmount}" done INOTIFY_PID=$( ( inotifywait -r -e modify -e move -e create -e delete -e attrib "${SNAPSHOT_DIR}" ${INOTIFY_EXCLUDES} 2>&1 & echo "PID: $!" ) | ( grep -m1 '^PID: ' | cut -d ' ' -f 2 && grep -q "Watches established" ) ) fi @@ -1240,7 +1260,7 @@ else log_error "ERROR: zypper ${ZYPPER_ARG} on ${MOUNT_DIR} failed with exit code ${RETVAL}!" if [ -n "${ZYPPER_NONINTERACTIVE}" ]; then - echo "Use '--interactive' for manual problem resolution." + log_error "Use '--interactive' for manual problem resolution." fi EXITCODE=1 fi @@ -1289,7 +1309,7 @@ fi if [ ${RUN_SHELL} -eq 1 ]; then - echo "Opening chroot in snapshot ${SNAPSHOT_ID}, continue with 'exit'" + log_info "Opening chroot in snapshot ${SNAPSHOT_ID}, continue with 'exit'" env PS1="transactional update # " chroot ${MOUNT_DIR} bash 2>&4 fi @@ -1365,8 +1385,7 @@ if [ ${EXITCODE} -ne 0 ]; then quit ${EXITCODE} elif [ $REBOOT_AFTERWARDS -eq 0 ]; then - echo - echo "Please reboot your machine to activate the changes and avoid data loss." + log_error "\nPlease reboot your machine to activate the changes and avoid data loss." touch "${NEEDS_RESTARTING_FILE}" fi @@ -1400,8 +1419,8 @@ reboot_autodetect ;; esac - echo "The system couldn't be rebooted using method '${REBOOT_METHOD}'. Please reboot the system" - echo "manually." + log_error "The system couldn't be rebooted using method '${REBOOT_METHOD}'. Please reboot the system" + log_error "manually." fi fi