From 6afb2f327a31e76e82dfa01dd33edca383a20c7f Mon Sep 17 00:00:00 2001 From: Paul Guo Date: Mon, 25 Jan 2021 15:23:33 +0800 Subject: [PATCH v4 2/2] Use copy_file_range() if possible for file copying in pg_rewind. We fallback for all if first call of copy_file_range() returns with errno ENOTSUP and EXDEV since this mostly implies subsequent calls of copy_file_range() on the same source and target would fail. --- configure | 2 +- configure.ac | 1 + src/bin/pg_rewind/file_ops.c | 54 ++++++++++++++++++++++++++++++++++++++++ src/bin/pg_rewind/file_ops.h | 4 +++ src/bin/pg_rewind/local_source.c | 15 +++++++++-- src/include/pg_config.h.in | 3 +++ src/tools/msvc/Solution.pm | 1 + 7 files changed, 77 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 7542fe30a1..74957e3980 100755 --- a/configure +++ b/configure @@ -15492,7 +15492,7 @@ fi LIBS_including_readline="$LIBS" LIBS=`echo "$LIBS" | sed -e 's/-ledit//g' -e 's/-lreadline//g'` -for ac_func in backtrace_symbols clock_gettime copyfile fdatasync getifaddrs getpeerucred getrlimit kqueue mbstowcs_l memset_s poll posix_fallocate ppoll pstat pthread_is_threaded_np readlink readv setproctitle setproctitle_fast setsid shm_open strchrnul strsignal symlink syncfs sync_file_range uselocale wcstombs_l writev +for ac_func in backtrace_symbols clock_gettime copyfile copy_file_range fdatasync getifaddrs getpeerucred getrlimit kqueue mbstowcs_l memset_s poll posix_fallocate ppoll pstat pthread_is_threaded_np readlink readv setproctitle setproctitle_fast setsid shm_open strchrnul strsignal symlink syncfs sync_file_range uselocale wcstombs_l writev do : as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" diff --git a/configure.ac b/configure.ac index ed3cdb9a8e..57e7233bab 100644 --- a/configure.ac +++ b/configure.ac @@ -1701,6 +1701,7 @@ AC_CHECK_FUNCS(m4_normalize([ backtrace_symbols clock_gettime copyfile + copy_file_range fdatasync getifaddrs getpeerucred diff --git a/src/bin/pg_rewind/file_ops.c b/src/bin/pg_rewind/file_ops.c index 0dd9c6c95e..f82b1d1250 100644 --- a/src/bin/pg_rewind/file_ops.c +++ b/src/bin/pg_rewind/file_ops.c @@ -39,6 +39,10 @@ static void remove_target_symlink(const char *path); static void recurse_dir(const char *datadir, const char *parentpath, process_file_callback_t callback); +#ifdef HAVE_COPY_FILE_RANGE +bool copy_file_range_support = true; +#endif + /* * Open a target file for writing. If 'trunc' is true and the file already * exists, it will be truncated. @@ -84,6 +88,56 @@ close_target_file(void) dstfd = -1; } +#ifdef HAVE_COPY_FILE_RANGE +/* + * Use copy_file_range() for file copy. Return true if succeeds, else return + * false. If copy_file_range() is not allowed between the source and target, + * set copy_file_range_support as false to prevent future calling of + * copy_file_range() for the source and target. + */ +bool +copy_target_range(int srcfd, off_t begin, size_t size) +{ + ssize_t copylen; + + /* update progress report */ + fetch_done += size; + progress_report(false); + + if (dry_run) + return true; + + if (lseek(srcfd, begin, SEEK_SET) == -1) + pg_fatal("could not seek in source file: %m"); + + if (lseek(dstfd, begin, SEEK_SET) == -1) + pg_fatal("could not seek in target file \"%s\": %m", dstpath); + + while (size > 0) + { + copylen = copy_file_range(srcfd, NULL, dstfd, NULL, size, 0); + + if (copylen <= 0) + { + /* + * Pre Linux 5.3 does not allow cross-fs copy_file_range() call + * (return EXDEV). Some fs do not support copy_file_range() (return + * ENOTSUP). Here we explicitly disable copy_file_range() for the + * two scenarios. For other failures we still allow subsequent + * copy_file_range() try. + */ + if (errno == ENOTSUP || errno == EXDEV) + copy_file_range_support = false; + return false; + } + + size -= copylen; + } + + return true; +} +#endif + void write_target_range(char *buf, off_t begin, size_t size) { diff --git a/src/bin/pg_rewind/file_ops.h b/src/bin/pg_rewind/file_ops.h index b6b8b319d5..a903ed54e0 100644 --- a/src/bin/pg_rewind/file_ops.h +++ b/src/bin/pg_rewind/file_ops.h @@ -13,6 +13,10 @@ #include "filemap.h" extern void open_target_file(const char *path, bool trunc); +#ifdef HAVE_COPY_FILE_RANGE +extern bool copy_file_range_support; +extern bool copy_target_range(int srcfd, off_t begin, size_t size); +#endif extern void write_target_range(char *buf, off_t begin, size_t size); extern void close_target_file(void); extern void remove_target_file(const char *path, bool missing_ok); diff --git a/src/bin/pg_rewind/local_source.c b/src/bin/pg_rewind/local_source.c index 9c3491c3fb..7abb3897ea 100644 --- a/src/bin/pg_rewind/local_source.c +++ b/src/bin/pg_rewind/local_source.c @@ -86,11 +86,22 @@ local_fetch_file_range(rewind_source *source, const char *path, off_t off, pg_fatal("could not open source file \"%s\": %m", srcpath); + open_target_file(path, false); + +#ifdef HAVE_COPY_FILE_RANGE + if (copy_file_range_support && + copy_target_range(srcfd, begin, end - begin)) + { + if (close(srcfd) != 0) + pg_fatal("could not close file \"%s\": %m", srcpath); + return; + } + /* else fall back to use write_target_range(). */ +#endif + if (lseek(srcfd, begin, SEEK_SET) == -1) pg_fatal("could not seek in source file: %m"); - open_target_file(path, false); - while (end - begin > 0) { ssize_t readlen; diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 15ffdd895a..edca394adb 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -104,6 +104,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_COPYFILE_H +/* Define to 1 if you have the `copy_file_range' function. */ +#undef HAVE_COPY_FILE_RANGE + /* Define to 1 if you have the header file. */ #undef HAVE_CRTDEFS_H diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index 165a93987a..f94f27a7ae 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -235,6 +235,7 @@ sub GenerateFiles HAVE_COMPUTED_GOTO => undef, HAVE_COPYFILE => undef, HAVE_COPYFILE_H => undef, + HAVE_COPY_FILE_RANGE => undef, HAVE_CRTDEFS_H => undef, HAVE_CRYPTO_LOCK => undef, HAVE_DECL_FDATASYNC => 0, -- 2.14.3