--- This change on top of the current preallocate branch implements the behavior I described of using fallocate if available (Linux syscall that uses filesystem-level preallocation support) or otherwise posix_fallocate if on Cygwin. This patch works on my computer (i.e., it makes the fallocate syscall), but it could use additional testing, especially on Cygwin.
Matt configure.in | 25 ++++++++++++++++++++++++- options.c | 2 +- receiver.c | 4 ++-- rsync.h | 3 ++- rsync.yo | 15 +++++++-------- syscall.c | 20 ++++++++++++++++++++ util.c | 6 +++--- 7 files changed, 59 insertions(+), 16 deletions(-) diff --git a/configure.in b/configure.in index c8e9394..36c9007 100644 --- a/configure.in +++ b/configure.in @@ -552,13 +552,36 @@ AC_CHECK_FUNCS(waitpid wait4 getcwd strdup chown chmod lchmod mknod mkfifo \ strlcat strlcpy strtol mallinfo getgroups setgroups geteuid getegid \ setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \ strerror putenv iconv_open locale_charset nl_langinfo getxattr \ - extattr_get_link sigaction sigprocmask setattrlist posix_fallocate) + extattr_get_link sigaction sigprocmask setattrlist fallocate posix_fallocate) dnl cygwin iconv.h defines iconv_open as libiconv_open if test x"$ac_cv_func_iconv_open" != x"yes"; then AC_CHECK_FUNC(libiconv_open, [ac_cv_func_iconv_open=yes; AC_DEFINE(HAVE_ICONV_OPEN, 1)]) fi +dnl Preallocation stuff (also fallocate, posix_fallocate function tests above): + +AC_CACHE_CHECK([for SYS_fallocate],rsync_cv_have_sys_fallocate,[ +AC_TRY_COMPILE([#include <sys/syscall.h> +#include <sys/types.h>], +[syscall(SYS_fallocate, 0, 0, (loff_t) 0, (loff_t) 0);], +rsync_cv_have_sys_fallocate=yes,rsync_cv_have_sys_fallocate=no)]) +if test x"$rsync_cv_have_sys_fallocate" = x"yes"; then + AC_DEFINE(HAVE_SYS_FALLOCATE, 1, [Define to 1 if you have the SYS_fallocate syscall number]) +fi + +dnl Copied from SHUTDOWN_ALL_SOCKETS test +AC_MSG_CHECKING([whether posix_fallocate is efficient]) +case $host_os in + *cygwin* ) AC_MSG_RESULT(yes) + AC_DEFINE(POSIX_FALLOCATE_IS_EFFICIENT, 1, + [Define if posix_fallocate is efficient (Cygwin)]) + ;; + * ) AC_MSG_RESULT(no);; +esac + +dnl End of preallocation stuff + AC_CHECK_FUNCS(getpgrp tcgetpgrp) if test $ac_cv_func_getpgrp = yes; then AC_FUNC_GETPGRP diff --git a/options.c b/options.c index 460966c..478e315 100644 --- a/options.c +++ b/options.c @@ -363,7 +363,7 @@ void usage(enum logcode F) #endif rprintf(F," -S, --sparse handle sparse files efficiently\n"); #ifdef SUPPORT_PREALLOCATION - rprintf(F," --preallocate posix_fallocate dest files before writing them\n"); + rprintf(F," --preallocate allocate dest files before writing them\n"); #endif rprintf(F," -n, --dry-run perform a trial run with no changes made\n"); rprintf(F," -W, --whole-file copy files whole (without delta-xfer algorithm)\n"); diff --git a/receiver.c b/receiver.c index 96c3c5e..a62ece0 100644 --- a/receiver.c +++ b/receiver.c @@ -182,10 +182,10 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, if (preallocate_files && fd != -1 && total_size > 0) { /* Preallocate enough space for file's eventual length if * possible; seems to reduce fragmentation on Windows. */ - if (posix_fallocate(fd, 0, total_size) == 0) + if (do_fallocate(fd, 0, total_size) == 0) preallocated_len = total_size; else - rsyserr(FINFO, errno, "preallocate %s", full_fname(fname)); + rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(fname)); } #endif diff --git a/rsync.h b/rsync.h index a5489f2..aeed22d 100644 --- a/rsync.h +++ b/rsync.h @@ -602,7 +602,8 @@ struct ht_int64_node { #define ACLS_NEED_MASK 1 #endif -#if defined HAVE_FTRUNCATE && defined HAVE_POSIX_FALLOCATE +#if HAVE_FTRUNCATE && (HAVE_FALLOCATE || HAVE_SYS_FALLOCATE \ + || (HAVE_POSIX_FALLOCATE && POSIX_FALLOCATE_IS_EFFICIENT)) #define SUPPORT_PREALLOCATION 1 #endif diff --git a/rsync.yo b/rsync.yo index b2ff1ce..ce0566f 100644 --- a/rsync.yo +++ b/rsync.yo @@ -352,7 +352,7 @@ to the detailed description below for a complete description. verb( --super receiver attempts super-user activities --fake-super store/recover privileged attrs using xattrs -S, --sparse handle sparse files efficiently - --preallocate posix_fallocate dest files before writing + --preallocate allocate dest files before writing -n, --dry-run perform a trial run with no changes made -W, --whole-file copy files whole (w/o delta-xfer algorithm) -x, --one-file-system don't cross filesystem boundaries @@ -1038,17 +1038,16 @@ filesystem. It doesn't seem to handle seeks over null regions correctly and ends up corrupting the files. dit(bf(--preallocate)) This tells the receiver to allocate each destination -file to its eventual size using bf(posix_fallocate)(3) before writing data -to the file. If the receiver is remote, this nonstandard option only works -if the receiver also has the preallocation patch. Furthermore, this option -only works if the receiver found the bf(posix_fallocate)(3) call at -configure time. +file to its eventual size before writing data to the file. Rsync will only use +the real filesystem-level preallocation support provided by bf(fallocate)(2) or +Cygwin's bf(posix_fallocate)(3), not the slow glibc implementation that writes +a zero byte into each block. If the receiver is remote, this nonstandard +option only works if the receiver also has the preallocation patch. Without this option on MS Windows, very large destination files tend to be broken into thousands of fragments; advising Windows ahead of time of the eventual file size using this option usually reduces the number of -fragments to one. However, on Linux, this option appears to just waste -disk I/O. +fragments to one. The usefulness of this option on Linux is yet to be tested. dit(bf(-n, --dry-run)) This makes rsync perform a trial run that doesn't make any changes (and produces mostly the same output as a real run). It diff --git a/syscall.c b/syscall.c index 39554e9..36f10b4 100644 --- a/syscall.c +++ b/syscall.c @@ -29,6 +29,10 @@ #include <sys/attr.h> #endif +#if !HAVE_FALLOCATE && HAVE_SYS_FALLOCATE +#include <sys/syscall.h> +#endif + extern int dry_run; extern int am_root; extern int read_only; @@ -291,3 +295,19 @@ char *d_name(struct dirent *di) return di->d_name; #endif } + +#ifdef SUPPORT_PREALLOCATION +int do_fallocate(int fd, OFF_T offset, OFF_T length) +{ + RETURN_ERROR_IF(dry_run, 0); + RETURN_ERROR_IF_RO_OR_LO; + /* TODO: Use FALLOC_FL_KEEP_SIZE to avoid the need to truncate. */ +#if HAVE_FALLOCATE + return fallocate(fd, 0, offset, length); +#elif HAVE_SYS_FALLOCATE + return syscall(SYS_fallocate, fd, 0, (loff_t) offset, (loff_t) length); +#else /* HAVE_POSIX_FALLOCATE && POSIX_FALLOCATE_IS_EFFICIENT */ + return posix_fallocate(fd, offset, length); +#endif +} +#endif diff --git a/util.c b/util.c index 2603e76..43c984f 100644 --- a/util.c +++ b/util.c @@ -306,13 +306,13 @@ int copy_file(const char *source, const char *dest, int ofd, STRUCT_STAT srcst; if (do_fstat(ifd, &srcst) == 0) { if (srcst.st_size > 0) { - if (posix_fallocate(ofd, 0, srcst.st_size) == 0) + if (do_fallocate(ofd, 0, srcst.st_size) == 0) preallocated_len = srcst.st_size; else - rsyserr(FINFO, errno, "posix_fallocate %s", full_fname(dest)); + rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(dest)); } } else - rsyserr(FINFO, errno, "fstat %s", full_fname(source)); + rsyserr(FWARNING, errno, "fstat %s", full_fname(source)); } #endif -- 1.5.4.3.193.g6dd0e -- To unsubscribe or change options: https://lists.samba.org/mailman/listinfo/rsync Before posting, read: http://www.catb.org/~esr/faqs/smart-questions.html