Package: automysqlbackup Version: 2.6+debian.4-3 Severity: normal Tags: patch
Combining the COMP=bzip2 and COMPDIRECT=yes arguments results in gzip compressed database dumps being created, but with ".bz2" file suffixes. I've created a patch which fixes this, and also adds support for pigz, zstd, lz4, pxz and other compression utilities. It also uses the `--reflink=auto` option in recent version of the cp program to make reflink copies of files when the `LATEST=yes` option is set. This may also fix #769656 and #783778. -- System Information: Debian Release: 10.9 APT prefers stable-debug APT policy: (500, 'stable-debug'), (500, 'stable'), (90, 'experimental'), (90, 'unstable') Architecture: amd64 (x86_64) Foreign Architectures: i386 Kernel: Linux 5.10.0-6-amd64 (SMP w/4 CPU cores) Kernel taint flags: TAINT_WARN Locale: LANG=en_GB.UTF-8, LC_CTYPE=en_GB.UTF-8 (charmap=UTF-8), LANGUAGE=en_GB.UTF-8 (charmap=UTF-8) Shell: /bin/sh linked to /bin/dash Init: systemd (via /run/systemd/system) LSM: AppArmor: enabled Versions of packages automysqlbackup depends on: ii bsd-mailx [mailx] 8.1.2-0.20180807cvs-1 ii default-mysql-client 1.0.5 ii mariadb-client-10.3 [virtual-mysql-client] 1:10.3.27-0+deb10u1 Versions of packages automysqlbackup recommends: ii mutt 1.10.1-2.1+deb10u5 automysqlbackup suggests no packages.
--- /usr/sbin/automysqlbackup 2020-07-04 19:56:25.000000000 +0100 +++ automysqlbackup.new 2021-05-21 11:56:11.448278767 +0100 @@ -80,10 +80,35 @@ # Two digit required DOMONTHLY=01 -# Choose Compression type. (gzip or bzip2) +# Compression program to use when compressing the output (gzip, pigz, bzip2, +# pbzip2, xz, pxz, pixz, lz4, zstd or zstdmt). The following published +# benchmark results may be useful to get an impression of relative compression +# ratios, and compression speed: +# https://community.centminmod.com/threads/round-4-compression-comparison-benchmarks-zstd-vs-brotli-vs-pigz-vs-bzip2-vs-xz-etc.18669/ +# e.g. "zstd -3" achieves slightly higher compression than "gzip -9", (3.17x vs +# 3.13x) but does so at 35x the throughput (449 MB/s vs. 12.7 MB/s) on the test +# system and dataset. COMP=gzip -# Compress backups on the fly with gzip or bzip2 (yes or no) +# Optional additional arguments to pass to the compression program (e.g. +# compression level, if you wish to override the default compression level for +# the selected utility). +#COMP_ARGS=("-9") + +# Print information about compression achieved. +COMP_VERBOSE="yes" + +# Compress backups on the fly, instead of writing uncompressed backups to disk +# first (yes or no). This requires less disk space (since it does not require +# that there be sufficient disk space available to store an additional +# uncompressed copy of each database prior to it being compressed). +# +# The use of "COMPDIRECT=yes" may also result in less time spent with the +# database locked (in situations when compressing on-the-fly and writing the +# compressed output to disk is quicker than writing uncompressed data to disk). +# This is most likely to be true on systems with multiple CPU cores when using +# compression utilities which are capable of parallel execution, and/or when +# using CPU-efficient compression algorithms such as zstd. COMPDIRECT=no # Compress communications between backup server and MySQL server? @@ -190,9 +215,9 @@ # set the DOWEEKLY setting, this can be a value from 1 to 7 where 1 is Monday, # The default is 6 which means that weekly backups are done on a Saturday. # -# COMP is used to choose the copmression used, options are gzip or bzip2. -# bzip2 will produce slightly smaller files but is more processor intensive so -# may take longer to complete. +# COMP is used to choose the compression used, options are gzip, bzip2, xz or +# zstd. On many systems and source data, zstd gives a good trade-off between +# file size and performance. # # COMMCOMP is used to enable or diable mysql client to server compression, so # it is useful to save bandwidth when backing up a remote MySQL server over @@ -430,71 +455,191 @@ # Functions -SUFFIX="" - # Database dump function dbdump () { + umask 077 + if [ "$1" = "information_schema" ] ; then NEWOPT="--skip-opt ${OPT}" else NEWOPT="--opt $OPT $OPTIONS" fi - if [ "$COMPDIRECT" = "yes" ] && ( [ "$COMP" = "gzip" ] || [ "$COMP" = "bzip2" ] ); then - if [ "$COMP" = "gzip" ]; then + if [ -z "${USERNAME}" -o -z "${PASSWORD}" ] ; then + USER_ARGS=("--defaults-file=/etc/mysql/debian.cnf") + else + USER_ARGS=("--user=$USERNAME" "--password=$PASSWORD" "--host=$DBHOST") + fi + + if [ "$COMPDIRECT" = "yes" ]; then + mysqldump ${USER_ARGS} $NEWOPT $1 | compression $2 --stdin + else + mysqldump ${USER_ARGS} $NEWOPT $1 > $2 + compression $2 + fi + return 0 +} + +# Compression function plus latest copy +# +# Takes the following arguments - either: +# +# . One argument - the path of a file on disk which should be compressed. The +# compressed data will be written to a file of the same name, with an +# appropriate suffix appended. If the compression is successful, then the +# original file will be removed. +# +# . Two arguments - first argument is the path to file to write to (WITHOUT the +# expected file suffix indicating the compression format used), and a second +# argument which is "--stdin" In this case, the standard input file descriptor +# is read, and the corresponding compressed data is written to the path +# specified. +# +# Uses the COMP_ARGS previously setup by the compression_prep() function. +compression () { + if [ "$COMP_VERBOSE" = "yes" ] ; then + echo Compression information for "${1}${SUFFIX}" + fi + + # Create a temporary file for the stderr of the compression command + # (which we will either echo to stdout or stderr depending on the + # return code of the command. + + tmpfile=$(mktemp /tmp/mysqldump.XXXXXX) + + if [ $# -eq 2 ] && [ $2 == "--stdin" ] ; then + #echo "DEBUG: executing: $COMP ${COMP_ARGS[@]} > ${1}{SUFFIX}" + $COMP ${COMP_ARGS[@]} > ${1}${SUFFIX} 2> $tmpfile + else + #echo "DEBUG: executing: $COMP ${COMP_ARGS[@]} $1" + $COMP ${COMP_ARGS[@]} $1 2> $tmpfile + fi + COMP_RC=$? + + # Compression utilities output verbose stats to stderr. In order to + # accommodate the logging convention of automysqlbackup, output the + # stderr of the compression command to either stdout of stderr of this + # function as appropriate. + + if [ $COMP_RC != 0 ] ; then + cat < $tmpfile >&2 + return $COMP_RC + else + cat < $tmpfile + fi + rm "$tmpfile" + + # Some compression utilities require an extra invocation to display + # information about the output file. + + if [ ${#COMP_ADDITIONAL_INVOCATION[@]} -ne 0 ] ; then + #echo "DEBUG: executing: ${COMP_ADDITIONAL_INVOCATION[@]} ${1}${SUFFIX}" + ${COMP_ADDITIONAL_INVOCATION[@]} "${1}${SUFFIX}" + fi + + if [ "$LATEST" = "yes" ] ; then + cp --reflink=auto $1$SUFFIX "$BACKUPDIR/latest/" + fi + + return $COMP_RC +} + + +# Derive compression arguments etc. which will be used by all invocations of +# compress(). +compression_prep () { + case $COMP in + gzip) SUFFIX=".gz" - touch "$2.gz" - chmod 600 "$2.gz" - if [ -z "${USERNAME}" -o -z "${PASSWORD}" ] ; then - mysqldump --defaults-file=/etc/mysql/debian.cnf $NEWOPT $1 | gzip -f > "$2.gz" - else - mysqldump --user=$USERNAME --password=$PASSWORD --host=$DBHOST $NEWOPT $1 | gzip -f > "$2.gz" + COMP_VERB_ARG="-v" + + # Default to compression level 9 for gzip for backward + # compatibility + + if [ ${#COMP_ARGS[@]} -eq 0 ]; then + COMP_ARGS=("-9") fi - elif [ "$COMP" = "bzip2" ]; then + ;; + pigz) + SUFFIX=".gz" + COMP_LIST_ARG="-l" + ;; + bzip2|pbzip2) SUFFIX=".bz2" - touch "$2.bz2" - chmod 600 "$2.bz2" - if [ -z "${USERNAME}" -o -z "${PASSWORD}" ] ; then - mysqldump --defaults-file=/etc/mysql/debian.cnf $NEWOPT $1 | gzip -f > "$2.bz2" - else - mysqldump --user=$USERNAME --password=$PASSWORD --host=$DBHOST $NEWOPT $1 | gzip -f > "$2.bz2" - fi - fi + COMP_VERB_ARG="-v" + ;; + lz4) + SUFFIX=".lz4" + COMP_RM_SRC_ARG="-m --rm" + COMP_VERB_ARG="-v" + ;; + xz) + SUFFIX=".xz" + COMP_VERB_ARG="-v" + ;; + pxz|pixz) + SUFFIX=".xz" + COMP_LIST_ARG="-l" + ;; + zstd|zstdmt) + SUFFIX=".zst" + COMP_RM_SRC_ARG="--rm" + COMP_LIST_ARG="-l" + COMP_ARGS+=("-q") + ;; + *) + echo "Unkown compression method (COMP='$COMP'), patches welcome!" + exit 1 + ;; + esac + + if [ "$COMPDIRECT" = "yes" ]; then + #COMP_ARGS+=("-c") # FIXME check if needed + true else - touch $2 - chmod 600 $2 - if [ -z "${USERNAME}" -o -z "${PASSWORD}" ] ; then - mysqldump --defaults-file=/etc/mysql/debian.cnf $NEWOPT $1 > $2 - compression $2 - else - mysqldump --user=$USERNAME --password=$PASSWORD --host=$DBHOST $NEWOPT $1 > $2 - compression $2 + if [ "COMP_RM_SRC_ARG" != "" ] ; then + #echo "DEBUG: adding comp rm src arg: $COMP_RM_SRC_ARG" + COMP_ARGS+=("$COMP_RM_SRC_ARG") fi fi - if [ "$LATEST" = "yes" ]; then - cp $1$SUFFIX "$BACKUPDIR/latest/" + + if [ "$COMP_VERBOSE" = "yes" ]; then + if [ "$COMP_LIST_ARG" != "" ] ; then + COMP_ADDITIONAL_INVOCATION=("$COMP" "$COMP_LIST_ARG") + fi + if [ "$COMP_VERB_ARG" != "" ] ; then + COMP_ARGS+=("$COMP_VERB_ARG") + fi + else + if [ "$COMP_QUIET_ARG" != "" ] ; then + COMP_ARGS+=("$COMP_QUIET_ARG") + fi fi - return 0 } -# Compression function plus latest copy -compression () { -if [ "$COMP" = "gzip" ]; then - gzip -f "$1" - echo - echo Backup Information for "$1" - gzip -l "$1.gz" - SUFFIX=".gz" -elif [ "$COMP" = "bzip2" ]; then - echo Compression information for "$1.bz2" - bzip2 -f -v $1 2>&1 - SUFFIX=".bz2" -else - echo "No compression option set, check advanced settings" -fi -return 0 +# This is not currently called, but documented here for developers wishing to +# maintain or extend the compression support. +compression_test () { + for COMP in gzip pigz bzip2 pbzip2 xz pixz lz4 zstd zstdmt ; do + for COMPDIRECT in yes no ; do + for COMP_VERBOSE in yes no ; do + ( cat /etc/default/automysqlbackup.in + echo "COMP=${COMP}" + echo "COMPDIRECT=${COMPDIRECT}" + echo "COMP_VERBOSE=${COMP_VERBOSE}" ) > /etc/default/automysqlbackup + echo + echo + echo '*****' + echo "COMP=${COMP} COMPDIRECT=${COMPDIRECT} COMP_VERBOSE=${COMP_VERBOSE}" + echo '*****' + ./automysqlbackup + done + done + done 2>&1 | tee comp_test_out } +# Evaluate the required compression utility arguments for later repeated use. +compression_prep # Run command before we begin if [ "$PREBACKUP" ]
