Package: automysqlbackup
Version: 2.6+debian.4-3
Severity: normal
Tags: patch

Combining the COMP=bzip2 and COMPDIRECT=yes arguments results in gzip
compressed database dumps being created, but with ".bz2" file suffixes.

I've created a patch which fixes this, and also adds support for pigz,
zstd, lz4, pxz and other compression utilities.

It also uses the `--reflink=auto` option in recent version of the cp
program to make reflink copies of files when the `LATEST=yes` option is
set.

This may also fix #769656 and #783778.


-- System Information:
Debian Release: 10.9
  APT prefers stable-debug
  APT policy: (500, 'stable-debug'), (500, 'stable'), (90, 'experimental'), 
(90, 'unstable')
Architecture: amd64 (x86_64)
Foreign Architectures: i386

Kernel: Linux 5.10.0-6-amd64 (SMP w/4 CPU cores)
Kernel taint flags: TAINT_WARN
Locale: LANG=en_GB.UTF-8, LC_CTYPE=en_GB.UTF-8 (charmap=UTF-8), 
LANGUAGE=en_GB.UTF-8 (charmap=UTF-8)
Shell: /bin/sh linked to /bin/dash
Init: systemd (via /run/systemd/system)
LSM: AppArmor: enabled

Versions of packages automysqlbackup depends on:
ii  bsd-mailx [mailx]                           8.1.2-0.20180807cvs-1
ii  default-mysql-client                        1.0.5
ii  mariadb-client-10.3 [virtual-mysql-client]  1:10.3.27-0+deb10u1

Versions of packages automysqlbackup recommends:
ii  mutt  1.10.1-2.1+deb10u5

automysqlbackup suggests no packages.
--- /usr/sbin/automysqlbackup   2020-07-04 19:56:25.000000000 +0100
+++ automysqlbackup.new 2021-05-21 11:56:11.448278767 +0100
@@ -80,10 +80,35 @@
 # Two digit required
 DOMONTHLY=01
 
-# Choose Compression type. (gzip or bzip2)
+# Compression program to use when compressing the output (gzip, pigz, bzip2,
+# pbzip2, xz, pxz, pixz, lz4, zstd or zstdmt).  The following published
+# benchmark results may be useful to get an impression of relative compression
+# ratios, and compression speed:
+# 
https://community.centminmod.com/threads/round-4-compression-comparison-benchmarks-zstd-vs-brotli-vs-pigz-vs-bzip2-vs-xz-etc.18669/
+# e.g. "zstd -3" achieves slightly higher compression than "gzip -9", (3.17x vs
+# 3.13x) but does so at 35x the throughput (449 MB/s vs. 12.7 MB/s) on the test
+# system and dataset.
 COMP=gzip
 
-# Compress backups on the fly with gzip or bzip2 (yes or no)
+# Optional additional arguments to pass to the compression program (e.g.
+# compression level, if you wish to override the default compression level for
+# the selected utility).
+#COMP_ARGS=("-9")
+
+# Print information about compression achieved.
+COMP_VERBOSE="yes"
+
+# Compress backups on the fly, instead of writing uncompressed backups to disk
+# first (yes or no).  This requires less disk space (since it does not require
+# that there be sufficient disk space available to store an additional
+# uncompressed copy of each database prior to it being compressed).
+#
+# The use of "COMPDIRECT=yes" may also result in less time spent with the
+# database locked (in situations when compressing on-the-fly and writing the
+# compressed output to disk is quicker than writing uncompressed data to disk).
+# This is most likely to be true on systems with multiple CPU cores when using
+# compression utilities which are capable of parallel execution, and/or when
+# using CPU-efficient compression algorithms such as zstd.
 COMPDIRECT=no
 
 # Compress communications between backup server and MySQL server?
@@ -190,9 +215,9 @@
 # set the DOWEEKLY setting, this can be a value from 1 to 7 where 1 is Monday,
 # The default is 6 which means that weekly backups are done on a Saturday.
 #
-# COMP is used to choose the copmression used, options are gzip or bzip2.
-# bzip2 will produce slightly smaller files but is more processor intensive so
-# may take longer to complete.
+# COMP is used to choose the compression used, options are gzip, bzip2, xz or
+# zstd.  On many systems and source data, zstd gives a good trade-off between
+# file size and performance.
 #
 # COMMCOMP is used to enable or diable mysql client to server compression, so
 # it is useful to save bandwidth when backing up a remote MySQL server over
@@ -430,71 +455,191 @@
 
 # Functions
 
-SUFFIX=""
-
 # Database dump function
 dbdump () {
+       umask 077
+
        if [ "$1" = "information_schema" ] ; then
                NEWOPT="--skip-opt ${OPT}"
        else
                NEWOPT="--opt $OPT $OPTIONS"
        fi
 
-       if [ "$COMPDIRECT" = "yes" ] && ( [ "$COMP" = "gzip" ] || [ "$COMP" = 
"bzip2" ] ); then
-               if [ "$COMP" = "gzip" ]; then
+       if [ -z "${USERNAME}" -o -z "${PASSWORD}" ] ; then
+               USER_ARGS=("--defaults-file=/etc/mysql/debian.cnf")
+       else
+               USER_ARGS=("--user=$USERNAME" "--password=$PASSWORD" 
"--host=$DBHOST")
+       fi
+
+       if [ "$COMPDIRECT" = "yes" ]; then
+               mysqldump ${USER_ARGS} $NEWOPT $1 | compression $2 --stdin
+       else
+               mysqldump ${USER_ARGS} $NEWOPT $1 > $2
+               compression $2
+       fi
+       return 0
+}
+
+# Compression function plus latest copy
+#
+# Takes the following arguments - either:
+#
+# . One argument - the path of a file on disk which should be compressed.  The
+# compressed data will be written to a file of the same name, with an
+# appropriate suffix appended.  If the compression is successful, then the
+# original file will be removed.
+#
+# . Two arguments - first argument is the path to file to write to (WITHOUT the
+# expected file suffix indicating the compression format used), and a second
+# argument which is "--stdin" In this case, the standard input file descriptor
+# is read, and the corresponding compressed data is written to the path
+# specified.
+#
+# Uses the COMP_ARGS previously setup by the compression_prep() function.
+compression () {
+       if [ "$COMP_VERBOSE" = "yes" ] ; then
+               echo Compression information for "${1}${SUFFIX}"
+       fi
+
+       # Create a temporary file for the stderr of the compression command
+       # (which we will either echo to stdout or stderr depending on the
+       # return code of the command.
+       
+       tmpfile=$(mktemp /tmp/mysqldump.XXXXXX)
+
+       if [ $# -eq 2 ] && [ $2 == "--stdin" ] ; then
+               #echo "DEBUG: executing: $COMP ${COMP_ARGS[@]} > ${1}{SUFFIX}"
+               $COMP ${COMP_ARGS[@]} > ${1}${SUFFIX} 2> $tmpfile
+       else
+               #echo "DEBUG: executing: $COMP ${COMP_ARGS[@]} $1"
+               $COMP ${COMP_ARGS[@]} $1 2> $tmpfile
+       fi
+       COMP_RC=$?
+
+       # Compression utilities output verbose stats to stderr.  In order to
+       # accommodate the logging convention of automysqlbackup, output the
+       # stderr of the compression command to either stdout of stderr of this
+       # function as appropriate.
+
+       if [ $COMP_RC != 0 ] ; then
+               cat < $tmpfile >&2
+               return $COMP_RC
+       else
+               cat < $tmpfile
+       fi
+       rm "$tmpfile"
+
+       # Some compression utilities require an extra invocation to display
+       # information about the output file.
+
+       if [ ${#COMP_ADDITIONAL_INVOCATION[@]} -ne 0 ] ; then
+               #echo "DEBUG: executing: ${COMP_ADDITIONAL_INVOCATION[@]} 
${1}${SUFFIX}"
+               ${COMP_ADDITIONAL_INVOCATION[@]} "${1}${SUFFIX}"
+       fi
+
+       if [ "$LATEST" = "yes" ] ; then
+               cp --reflink=auto $1$SUFFIX "$BACKUPDIR/latest/"
+       fi      
+
+       return $COMP_RC
+}
+
+
+# Derive compression arguments etc. which will be used by all invocations of
+# compress().
+compression_prep () {
+       case $COMP in
+               gzip)
                        SUFFIX=".gz"
-                       touch "$2.gz"
-                       chmod 600 "$2.gz"
-                       if [ -z "${USERNAME}" -o -z "${PASSWORD}" ] ; then
-                               mysqldump --defaults-file=/etc/mysql/debian.cnf 
$NEWOPT $1 | gzip -f > "$2.gz"
-                       else
-                               mysqldump --user=$USERNAME --password=$PASSWORD 
--host=$DBHOST $NEWOPT $1  | gzip -f > "$2.gz"
+                       COMP_VERB_ARG="-v"
+
+                       # Default to compression level 9 for gzip for backward
+                       # compatibility
+
+                       if [ ${#COMP_ARGS[@]} -eq 0 ]; then
+                               COMP_ARGS=("-9")
                        fi
-               elif [ "$COMP" = "bzip2" ]; then
+                       ;;
+               pigz)
+                       SUFFIX=".gz"
+                       COMP_LIST_ARG="-l"
+                       ;;
+               bzip2|pbzip2)
                        SUFFIX=".bz2"
-                       touch "$2.bz2"
-                        chmod 600 "$2.bz2"
-                       if [ -z "${USERNAME}" -o -z "${PASSWORD}" ] ; then
-                                mysqldump 
--defaults-file=/etc/mysql/debian.cnf $NEWOPT $1 | gzip -f > "$2.bz2"
-                        else
-                                mysqldump --user=$USERNAME 
--password=$PASSWORD --host=$DBHOST $NEWOPT $1  | gzip -f > "$2.bz2"
-                        fi
-               fi
+                       COMP_VERB_ARG="-v"
+                       ;;
+               lz4)
+                       SUFFIX=".lz4"
+                       COMP_RM_SRC_ARG="-m --rm"
+                       COMP_VERB_ARG="-v"
+                       ;;
+               xz)
+                       SUFFIX=".xz"
+                       COMP_VERB_ARG="-v"
+                       ;;
+               pxz|pixz)
+                       SUFFIX=".xz"
+                       COMP_LIST_ARG="-l"
+                       ;;
+               zstd|zstdmt)
+                       SUFFIX=".zst"
+                       COMP_RM_SRC_ARG="--rm"
+                       COMP_LIST_ARG="-l"
+                       COMP_ARGS+=("-q")
+                       ;;
+               *)
+                       echo "Unkown compression method (COMP='$COMP'), patches 
welcome!"
+                       exit 1
+                       ;;
+       esac
+
+       if [ "$COMPDIRECT" = "yes" ]; then
+               #COMP_ARGS+=("-c")  # FIXME check if needed
+               true
        else
-               touch $2
-               chmod 600 $2
-               if [ -z "${USERNAME}" -o -z "${PASSWORD}" ] ; then
-                       mysqldump --defaults-file=/etc/mysql/debian.cnf $NEWOPT 
$1 > $2
-                       compression $2
-               else
-                       mysqldump --user=$USERNAME --password=$PASSWORD 
--host=$DBHOST $NEWOPT $1 > $2
-                       compression $2
+               if [ "COMP_RM_SRC_ARG" != "" ] ; then
+                       #echo "DEBUG: adding comp rm src arg: $COMP_RM_SRC_ARG"
+                       COMP_ARGS+=("$COMP_RM_SRC_ARG")
                fi
        fi
-       if [ "$LATEST" = "yes" ]; then
-               cp $1$SUFFIX "$BACKUPDIR/latest/"
+
+       if [ "$COMP_VERBOSE" = "yes" ]; then
+               if [ "$COMP_LIST_ARG" != "" ] ; then
+                       COMP_ADDITIONAL_INVOCATION=("$COMP" "$COMP_LIST_ARG")
+               fi
+               if [ "$COMP_VERB_ARG" != "" ] ; then
+                       COMP_ARGS+=("$COMP_VERB_ARG")
+               fi
+       else
+               if [ "$COMP_QUIET_ARG" != "" ] ; then
+                       COMP_ARGS+=("$COMP_QUIET_ARG")
+               fi
        fi
-       return 0
 }
 
-# Compression function plus latest copy
-compression () {
-if [ "$COMP" = "gzip" ]; then
-       gzip -f "$1"
-       echo
-       echo Backup Information for "$1"
-       gzip -l "$1.gz"
-       SUFFIX=".gz"
-elif [ "$COMP" = "bzip2" ]; then
-       echo Compression information for "$1.bz2"
-       bzip2 -f -v $1 2>&1
-       SUFFIX=".bz2"
-else
-       echo "No compression option set, check advanced settings"
-fi
-return 0
+# This is not currently called, but documented here for developers wishing to
+# maintain or extend the compression support.
+compression_test () {
+       for COMP in gzip pigz bzip2 pbzip2 xz pixz lz4 zstd zstdmt ; do 
+               for COMPDIRECT in yes no ; do
+                       for COMP_VERBOSE in yes no ; do
+                               ( cat /etc/default/automysqlbackup.in
+                               echo "COMP=${COMP}"
+                               echo "COMPDIRECT=${COMPDIRECT}"
+                               echo "COMP_VERBOSE=${COMP_VERBOSE}" ) > 
/etc/default/automysqlbackup
+                               echo
+                               echo
+                               echo '*****'
+                               echo "COMP=${COMP} COMPDIRECT=${COMPDIRECT} 
COMP_VERBOSE=${COMP_VERBOSE}"
+                               echo '*****'
+                               ./automysqlbackup
+                       done
+               done
+       done 2>&1 | tee comp_test_out
 }
 
+# Evaluate the required compression utility arguments for later repeated use.
+compression_prep
 
 # Run command before we begin
 if [ "$PREBACKUP" ]

Reply via email to