From a5301655dd927e2c37c0b6156c24568bdba6eac3 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Fri, 13 May 2016 14:27:04 +0900 Subject: [PATCH 1/2] Relocation fsync routines of initdb into fe_utils Those are aimed at being used by other utilities, pg_basebackup mainly, and at some other degree pg_dump and pg_receivexlog. --- src/bin/initdb/Makefile | 3 +- src/bin/initdb/initdb.c | 266 +----------------------------------- src/fe_utils/Makefile | 2 +- src/fe_utils/file_utils.c | 279 ++++++++++++++++++++++++++++++++++++++ src/include/fe_utils/file_utils.h | 22 +++ src/tools/msvc/Mkvcbuild.pm | 3 +- 6 files changed, 313 insertions(+), 262 deletions(-) create mode 100644 src/fe_utils/file_utils.c create mode 100644 src/include/fe_utils/file_utils.h diff --git a/src/bin/initdb/Makefile b/src/bin/initdb/Makefile index 094c894..e6b3f72 100644 --- a/src/bin/initdb/Makefile +++ b/src/bin/initdb/Makefile @@ -17,6 +17,7 @@ top_builddir = ../../.. include $(top_builddir)/src/Makefile.global override CPPFLAGS := -DFRONTEND -I$(libpq_srcdir) -I$(top_srcdir)/src/timezone $(CPPFLAGS) +LDFLAGS += -L$(top_builddir)/src/fe_utils -lpgfeutils # use system timezone data? ifneq (,$(with_system_tzdata)) @@ -27,7 +28,7 @@ OBJS= initdb.o findtimezone.o localtime.o encnames.o $(WIN32RES) all: initdb -initdb: $(OBJS) | submake-libpgport +initdb: $(OBJS) | submake-libpgport submake-libpgfeutils $(CC) $(CFLAGS) $(OBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) # We used to pull in all of libpq to get encnames.c, but that diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index ec8c38e..9519902 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -63,19 +63,13 @@ #include "catalog/catalog.h" #include "common/restricted_token.h" #include "common/username.h" +#include "fe_utils/file_utils.h" #include "mb/pg_wchar.h" #include "getaddrinfo.h" #include "getopt_long.h" #include "miscadmin.h" -/* Define PG_FLUSH_DATA_WORKS if we have an implementation for pg_flush_data */ -#if defined(HAVE_SYNC_FILE_RANGE) -#define PG_FLUSH_DATA_WORKS 1 -#elif defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED) -#define PG_FLUSH_DATA_WORKS 1 -#endif - /* Ideally this would be in a .h file, but it hardly seems worth the trouble */ extern const char *select_default_timezone(const char *share_path); @@ -235,13 +229,6 @@ static char **filter_lines_with_token(char **lines, const char *token); #endif static char **readfile(const char *path); static void writefile(char *path, char **lines); -static void walkdir(const char *path, - void (*action) (const char *fname, bool isdir), - bool process_symlinks); -#ifdef PG_FLUSH_DATA_WORKS -static void pre_sync_fname(const char *fname, bool isdir); -#endif -static void fsync_fname_ext(const char *fname, bool isdir); static FILE *popen_check(const char *command, const char *mode); static void exit_nicely(void); static char *get_id(void); @@ -268,7 +255,6 @@ static void load_plpgsql(FILE *cmdfd); static void vacuum_db(FILE *cmdfd); static void make_template0(FILE *cmdfd); static void make_postgres(FILE *cmdfd); -static void fsync_pgdata(void); static void trapsig(int signum); static void check_ok(void); static char *escape_quotes(const char *src); @@ -535,177 +521,6 @@ writefile(char *path, char **lines) } /* - * walkdir: recursively walk a directory, applying the action to each - * regular file and directory (including the named directory itself). - * - * If process_symlinks is true, the action and recursion are also applied - * to regular files and directories that are pointed to by symlinks in the - * given directory; otherwise symlinks are ignored. Symlinks are always - * ignored in subdirectories, ie we intentionally don't pass down the - * process_symlinks flag to recursive calls. - * - * Errors are reported but not considered fatal. - * - * See also walkdir in fd.c, which is a backend version of this logic. - */ -static void -walkdir(const char *path, - void (*action) (const char *fname, bool isdir), - bool process_symlinks) -{ - DIR *dir; - struct dirent *de; - - dir = opendir(path); - if (dir == NULL) - { - fprintf(stderr, _("%s: could not open directory \"%s\": %s\n"), - progname, path, strerror(errno)); - return; - } - - while (errno = 0, (de = readdir(dir)) != NULL) - { - char subpath[MAXPGPATH]; - struct stat fst; - int sret; - - if (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0) - continue; - - snprintf(subpath, MAXPGPATH, "%s/%s", path, de->d_name); - - if (process_symlinks) - sret = stat(subpath, &fst); - else - sret = lstat(subpath, &fst); - - if (sret < 0) - { - fprintf(stderr, _("%s: could not stat file \"%s\": %s\n"), - progname, subpath, strerror(errno)); - continue; - } - - if (S_ISREG(fst.st_mode)) - (*action) (subpath, false); - else if (S_ISDIR(fst.st_mode)) - walkdir(subpath, action, false); - } - - if (errno) - fprintf(stderr, _("%s: could not read directory \"%s\": %s\n"), - progname, path, strerror(errno)); - - (void) closedir(dir); - - /* - * It's important to fsync the destination directory itself as individual - * file fsyncs don't guarantee that the directory entry for the file is - * synced. Recent versions of ext4 have made the window much wider but - * it's been an issue for ext3 and other filesystems in the past. - */ - (*action) (path, true); -} - -/* - * Hint to the OS that it should get ready to fsync() this file. - * - * Ignores errors trying to open unreadable files, and reports other errors - * non-fatally. - */ -#ifdef PG_FLUSH_DATA_WORKS - -static void -pre_sync_fname(const char *fname, bool isdir) -{ - int fd; - - fd = open(fname, O_RDONLY | PG_BINARY); - - if (fd < 0) - { - if (errno == EACCES || (isdir && errno == EISDIR)) - return; - fprintf(stderr, _("%s: could not open file \"%s\": %s\n"), - progname, fname, strerror(errno)); - return; - } - - /* - * We do what pg_flush_data() would do in the backend: prefer to use - * sync_file_range, but fall back to posix_fadvise. We ignore errors - * because this is only a hint. - */ -#if defined(HAVE_SYNC_FILE_RANGE) - (void) sync_file_range(fd, 0, 0, SYNC_FILE_RANGE_WRITE); -#elif defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED) - (void) posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); -#else -#error PG_FLUSH_DATA_WORKS should not have been defined -#endif - - (void) close(fd); -} - -#endif /* PG_FLUSH_DATA_WORKS */ - -/* - * fsync_fname_ext -- Try to fsync a file or directory - * - * Ignores errors trying to open unreadable files, or trying to fsync - * directories on systems where that isn't allowed/required. Reports - * other errors non-fatally. - */ -static void -fsync_fname_ext(const char *fname, bool isdir) -{ - int fd; - int flags; - int returncode; - - /* - * Some OSs require directories to be opened read-only whereas other - * systems don't allow us to fsync files opened read-only; so we need both - * cases here. Using O_RDWR will cause us to fail to fsync files that are - * not writable by our userid, but we assume that's OK. - */ - flags = PG_BINARY; - if (!isdir) - flags |= O_RDWR; - else - flags |= O_RDONLY; - - /* - * Open the file, silently ignoring errors about unreadable files (or - * unsupported operations, e.g. opening a directory under Windows), and - * logging others. - */ - fd = open(fname, flags); - if (fd < 0) - { - if (errno == EACCES || (isdir && errno == EISDIR)) - return; - fprintf(stderr, _("%s: could not open file \"%s\": %s\n"), - progname, fname, strerror(errno)); - return; - } - - returncode = fsync(fd); - - /* - * Some OSes don't allow us to fsync directories at all, so we can ignore - * those errors. Anything else needs to be reported. - */ - if (returncode != 0 && !(isdir && errno == EBADF)) - fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), - progname, fname, strerror(errno)); - - (void) close(fd); -} - -/* * Open a subcommand with suitable error messaging */ static FILE * @@ -2297,77 +2112,6 @@ make_postgres(FILE *cmdfd) PG_CMD_PUTS(*line); } -/* - * Issue fsync recursively on PGDATA and all its contents. - * - * We fsync regular files and directories wherever they are, but we - * follow symlinks only for pg_xlog and immediately under pg_tblspc. - * Other symlinks are presumed to point at files we're not responsible - * for fsyncing, and might not have privileges to write at all. - * - * Errors are reported but not considered fatal. - */ -static void -fsync_pgdata(void) -{ - bool xlog_is_symlink; - char pg_xlog[MAXPGPATH]; - char pg_tblspc[MAXPGPATH]; - - fputs(_("syncing data to disk ... "), stdout); - fflush(stdout); - - snprintf(pg_xlog, MAXPGPATH, "%s/pg_xlog", pg_data); - snprintf(pg_tblspc, MAXPGPATH, "%s/pg_tblspc", pg_data); - - /* - * If pg_xlog is a symlink, we'll need to recurse into it separately, - * because the first walkdir below will ignore it. - */ - xlog_is_symlink = false; - -#ifndef WIN32 - { - struct stat st; - - if (lstat(pg_xlog, &st) < 0) - fprintf(stderr, _("%s: could not stat file \"%s\": %s\n"), - progname, pg_xlog, strerror(errno)); - else if (S_ISLNK(st.st_mode)) - xlog_is_symlink = true; - } -#else - if (pgwin32_is_junction(pg_xlog)) - xlog_is_symlink = true; -#endif - - /* - * If possible, hint to the kernel that we're soon going to fsync the data - * directory and its contents. - */ -#ifdef PG_FLUSH_DATA_WORKS - walkdir(pg_data, pre_sync_fname, false); - if (xlog_is_symlink) - walkdir(pg_xlog, pre_sync_fname, false); - walkdir(pg_tblspc, pre_sync_fname, true); -#endif - - /* - * Now we do the fsync()s in the same order. - * - * The main call ignores symlinks, so in addition to specially processing - * pg_xlog if it's a symlink, pg_tblspc has to be visited separately with - * process_symlinks = true. Note that if there are any plain directories - * in pg_tblspc, they'll get fsync'd twice. That's not an expected case - * so we don't worry about optimizing it. - */ - walkdir(pg_data, fsync_fname_ext, false); - if (xlog_is_symlink) - walkdir(pg_xlog, fsync_fname_ext, false); - walkdir(pg_tblspc, fsync_fname_ext, true); - - check_ok(); -} /* @@ -3534,7 +3278,8 @@ main(int argc, char *argv[]) exit_nicely(); } - fsync_pgdata(); + fsync_pgdata(pg_data, progname); + check_ok(); return 0; } @@ -3593,7 +3338,10 @@ main(int argc, char *argv[]) initialize_data_directory(); if (do_sync) - fsync_pgdata(); + { + fsync_pgdata(pg_data, progname); + check_ok(); + } else printf(_("\nSync to disk skipped.\nThe data directory might become corrupt if the operating system crashes.\n")); diff --git a/src/fe_utils/Makefile b/src/fe_utils/Makefile index 9da03b1..51a7804 100644 --- a/src/fe_utils/Makefile +++ b/src/fe_utils/Makefile @@ -19,7 +19,7 @@ include $(top_builddir)/src/Makefile.global override CPPFLAGS := -DFRONTEND -I$(libpq_srcdir) $(CPPFLAGS) -OBJS = mbprint.o print.o psqlscan.o simple_list.o string_utils.o +OBJS = file_utils.o mbprint.o print.o psqlscan.o simple_list.o string_utils.o all: libpgfeutils.a diff --git a/src/fe_utils/file_utils.c b/src/fe_utils/file_utils.c new file mode 100644 index 0000000..80f6dbd --- /dev/null +++ b/src/fe_utils/file_utils.c @@ -0,0 +1,279 @@ +/*------------------------------------------------------------------------- + * + * File-processing utility routines for frontend code + * + * Assorted utility functions to work on files. + * + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/fe_utils/file_utils.c + * + *------------------------------------------------------------------------- + */ +#include "postgres_fe.h" + +#include +#include +#include +#include + +#include "fe_utils/file_utils.h" + + +/* Define PG_FLUSH_DATA_WORKS if we have an implementation for pg_flush_data */ +#if defined(HAVE_SYNC_FILE_RANGE) +#define PG_FLUSH_DATA_WORKS 1 +#elif defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED) +#define PG_FLUSH_DATA_WORKS 1 +#endif + +#ifdef PG_FLUSH_DATA_WORKS +static void pre_sync_fname(const char *fname, bool isdir, + const char *progname); +#endif +static void walkdir(const char *path, + void (*action) (const char *fname, bool isdir, const char *progname), + bool process_symlinks, const char *progname); + +/* + * Issue fsync recursively on PGDATA and all its contents. + * + * We fsync regular files and directories wherever they are, but we + * follow symlinks only for pg_xlog and immediately under pg_tblspc. + * Other symlinks are presumed to point at files we're not responsible + * for fsyncing, and might not have privileges to write at all. + * + * Errors are reported but not considered fatal. + */ +void +fsync_pgdata(const char *pg_data, const char *progname) +{ + bool xlog_is_symlink; + char pg_xlog[MAXPGPATH]; + char pg_tblspc[MAXPGPATH]; + + fputs(_("syncing data to disk ... "), stdout); + fflush(stdout); + + snprintf(pg_xlog, MAXPGPATH, "%s/pg_xlog", pg_data); + snprintf(pg_tblspc, MAXPGPATH, "%s/pg_tblspc", pg_data); + + /* + * If pg_xlog is a symlink, we'll need to recurse into it separately, + * because the first walkdir below will ignore it. + */ + xlog_is_symlink = false; + +#ifndef WIN32 + { + struct stat st; + + if (lstat(pg_xlog, &st) < 0) + fprintf(stderr, _("could not stat file \"%s\": %s\n"), + pg_xlog, strerror(errno)); + else if (S_ISLNK(st.st_mode)) + xlog_is_symlink = true; + } +#else + if (pgwin32_is_junction(pg_xlog)) + xlog_is_symlink = true; +#endif + + /* + * If possible, hint to the kernel that we're soon going to fsync the data + * directory and its contents. + */ +#ifdef PG_FLUSH_DATA_WORKS + walkdir(pg_data, pre_sync_fname, false, progname); + if (xlog_is_symlink) + walkdir(pg_xlog, pre_sync_fname, false, progname); + walkdir(pg_tblspc, pre_sync_fname, true, progname); +#endif + + /* + * Now we do the fsync()s in the same order. + * + * The main call ignores symlinks, so in addition to specially processing + * pg_xlog if it's a symlink, pg_tblspc has to be visited separately with + * process_symlinks = true. Note that if there are any plain directories + * in pg_tblspc, they'll get fsync'd twice. That's not an expected case + * so we don't worry about optimizing it. + */ + walkdir(pg_data, fsync_fname_ext, false, progname); + if (xlog_is_symlink) + walkdir(pg_xlog, fsync_fname_ext, false, progname); + walkdir(pg_tblspc, fsync_fname_ext, true, progname); +} + +/* + * walkdir: recursively walk a directory, applying the action to each + * regular file and directory (including the named directory itself). + * + * If process_symlinks is true, the action and recursion are also applied + * to regular files and directories that are pointed to by symlinks in the + * given directory; otherwise symlinks are ignored. Symlinks are always + * ignored in subdirectories, ie we intentionally don't pass down the + * process_symlinks flag to recursive calls. + * + * Errors are reported but not considered fatal. + * + * See also walkdir in fd.c, which is a backend version of this logic. + */ +static void +walkdir(const char *path, + void (*action) (const char *fname, bool isdir, const char *progname), + bool process_symlinks, const char *progname) +{ + DIR *dir; + struct dirent *de; + + dir = opendir(path); + if (dir == NULL) + { + fprintf(stderr, _("%s: could not open directory \"%s\": %s\n"), + progname, path, strerror(errno)); + return; + } + + while (errno = 0, (de = readdir(dir)) != NULL) + { + char subpath[MAXPGPATH]; + struct stat fst; + int sret; + + if (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0) + continue; + + snprintf(subpath, MAXPGPATH, "%s/%s", path, de->d_name); + + if (process_symlinks) + sret = stat(subpath, &fst); + else + sret = lstat(subpath, &fst); + + if (sret < 0) + { + fprintf(stderr, _("%s: could not stat file \"%s\": %s\n"), + progname, subpath, strerror(errno)); + continue; + } + + if (S_ISREG(fst.st_mode)) + (*action) (subpath, false, progname); + else if (S_ISDIR(fst.st_mode)) + walkdir(subpath, action, false, progname); + } + + if (errno) + fprintf(stderr, _("%s: could not read directory \"%s\": %s\n"), + progname, path, strerror(errno)); + + (void) closedir(dir); + + /* + * It's important to fsync the destination directory itself as individual + * file fsyncs don't guarantee that the directory entry for the file is + * synced. Recent versions of ext4 have made the window much wider but + * it's been an issue for ext3 and other filesystems in the past. + */ + (*action) (path, true, progname); +} + +/* + * Hint to the OS that it should get ready to fsync() this file. + * + * Ignores errors trying to open unreadable files, and reports other errors + * non-fatally. + */ +#ifdef PG_FLUSH_DATA_WORKS + +static void +pre_sync_fname(const char *fname, bool isdir, const char *progname) +{ + int fd; + + fd = open(fname, O_RDONLY | PG_BINARY); + + if (fd < 0) + { + if (errno == EACCES || (isdir && errno == EISDIR)) + return; + fprintf(stderr, _("%s: could not open file \"%s\": %s\n"), + progname, fname, strerror(errno)); + return; + } + + /* + * We do what pg_flush_data() would do in the backend: prefer to use + * sync_file_range, but fall back to posix_fadvise. We ignore errors + * because this is only a hint. + */ +#if defined(HAVE_SYNC_FILE_RANGE) + (void) sync_file_range(fd, 0, 0, SYNC_FILE_RANGE_WRITE); +#elif defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED) + (void) posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); +#else +#error PG_FLUSH_DATA_WORKS should not have been defined +#endif + + (void) close(fd); +} + +#endif /* PG_FLUSH_DATA_WORKS */ + +/* + * fsync_fname_ext -- Try to fsync a file or directory + * + * Ignores errors trying to open unreadable files, or trying to fsync + * directories on systems where that isn't allowed/required. Reports + * other errors non-fatally. + */ +void +fsync_fname_ext(const char *fname, bool isdir, const char *progname) +{ + int fd; + int flags; + int returncode; + + /* + * Some OSs require directories to be opened read-only whereas other + * systems don't allow us to fsync files opened read-only; so we need both + * cases here. Using O_RDWR will cause us to fail to fsync files that are + * not writable by our userid, but we assume that's OK. + */ + flags = PG_BINARY; + if (!isdir) + flags |= O_RDWR; + else + flags |= O_RDONLY; + + /* + * Open the file, silently ignoring errors about unreadable files (or + * unsupported operations, e.g. opening a directory under Windows), and + * logging others. + */ + fd = open(fname, flags); + if (fd < 0) + { + if (errno == EACCES || (isdir && errno == EISDIR)) + return; + fprintf(stderr, _("%s: could not open file \"%s\": %s\n"), + progname, fname, strerror(errno)); + return; + } + + returncode = fsync(fd); + + /* + * Some OSes don't allow us to fsync directories at all, so we can ignore + * those errors. Anything else needs to be reported. + */ + if (returncode != 0 && !(isdir && errno == EBADF)) + fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), + progname, fname, strerror(errno)); + + (void) close(fd); +} diff --git a/src/include/fe_utils/file_utils.h b/src/include/fe_utils/file_utils.h new file mode 100644 index 0000000..189cc6d --- /dev/null +++ b/src/include/fe_utils/file_utils.h @@ -0,0 +1,22 @@ +/*------------------------------------------------------------------------- + * + * File-processing utility routines for frontend code + * + * Assorted utility functions to work on files. + * + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/fe_utils/file_utils.h + * + *------------------------------------------------------------------------- + */ +#ifndef FILE_UTILS_H +#define FILE_UTILS_H + +extern void fsync_fname_ext(const char *fname, bool isdir, + const char *progname); +extern void fsync_pgdata(const char *pg_data, const char *progname); + +#endif /* FILE_UTILS_H */ diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index e7268cb..e09c29d 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -118,7 +118,8 @@ sub mkvcbuild our @pgcommonbkndfiles = @pgcommonallfiles; our @pgfeutilsfiles = qw( - mbprint.c print.c psqlscan.l psqlscan.c simple_list.c string_utils.c); + file_utils.c mbprint.c print.c psqlscan.l psqlscan.c simple_list.c + string_utils.c); $libpgport = $solution->AddProject('libpgport', 'lib', 'misc'); $libpgport->AddDefine('FRONTEND'); -- 2.8.2