Hey all, almost 48h ago I upgraded both my instances to 2.3.1 again to see if the new patches would fix the replication issues for me.
So far, the result is: great. I haven't been able to provoke any kind of I/O stall or persisting queued/failed resync requests in my replication setup. Newly added users are replicated instantly upon the first received mails and the home directory gets created without issues now too. For reference: I'm using the official 2.3.1 tarball together with the 3 attached patches, that have been taken from GitHub diffs/commits linked to me by Aki in the #dovecot channel. I can only encourage everyone to try out 2.3.1 again with these 3 patches to make sure it is rock-solid so that we might get a proper and stable 2.3.2 release soon-ish :) PS: For the Arch Linux users among you the dovecot-2.3.1-5 package in the official repo contains said three patches :) Cheers, Thore -- Thore Bödecker GPG ID: 0xD622431AF8DB80F3 GPG FP: 0F96 559D 3556 24FC 2226 A864 D622 431A F8DB 80F3
commit 890883f12e8d8dd3309743eb95cf0b04f6e39ea0 Author: Aki Tuomi <aki.tu...@dovecot.fi> Date: Mon Mar 19 18:39:27 2018 +0200 dsync: Revert to /tmp if home does not exist Fixes doveadm: Error: Couldn't lock .dovecot-sync.lock: safe_mkstemp(.dovecot-sync.lock) failed: No such file or directory diff --git a/src/doveadm/dsync/dsync-brain.c b/src/doveadm/dsync/dsync-brain.c index c2b8169..1e84182 100644 --- a/src/doveadm/dsync/dsync-brain.c +++ b/src/doveadm/dsync/dsync-brain.c @@ -401,6 +401,7 @@ dsync_brain_lock(struct dsync_brain *brain, const char *remote_hostname) .lock_method = FILE_LOCK_METHOD_FCNTL, }; const char *home, *error, *local_hostname = my_hostdomain(); + struct stat st; bool created; int ret; @@ -437,8 +438,21 @@ dsync_brain_lock(struct dsync_brain *brain, const char *remote_hostname) if (brain->verbose_proctitle) process_title_set(dsync_brain_get_proctitle_full(brain, DSYNC_BRAIN_TITLE_LOCKING)); - brain->lock_path = p_strconcat(brain->pool, home, - "/"DSYNC_LOCK_FILENAME, NULL); + + /* if homedir does not yet exist, create lock under tmpdir */ + if (stat(home, &st) < 0) { + if (errno != ENOENT) { + i_error("stat(%s) failed: %m", home); + return -1; + } + brain->lock_path = p_strdup_printf(brain->pool, "%s/%s-%s", + brain->user->set->mail_temp_dir, + brain->user->username, + "/"DSYNC_LOCK_FILENAME); + } else { + brain->lock_path = p_strconcat(brain->pool, home, + "/"DSYNC_LOCK_FILENAME, NULL); + } brain->lock_fd = file_create_locked(brain->lock_path, &lock_set, &brain->lock, &created, &error); if (brain->lock_fd == -1)
From a952e178943a5944255cb7c053d970f8e6d49336 Mon Sep 17 00:00:00 2001 From: Timo Sirainen <timo.sirai...@dovecot.fi> Date: Tue, 5 Jun 2018 20:23:52 +0300 Subject: [PATCH] doveadm-server: Fix hang when sending a lot of output to clients Nowadays ostream adds its io to the stream's specified ioloop, not to current ioloop. --- src/doveadm/client-connection-tcp.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/doveadm/client-connection-tcp.c b/src/doveadm/client-connection-tcp.c index a2e1358d7f..672017495d 100644 --- a/src/doveadm/client-connection-tcp.c +++ b/src/doveadm/client-connection-tcp.c @@ -336,6 +336,9 @@ static int doveadm_cmd_handle(struct client_connection_tcp *conn, running one and we can't call the original one recursively, so create a new ioloop. */ conn->ioloop = io_loop_create(); + o_stream_switch_ioloop(conn->output); + if (conn->log_out != NULL) + o_stream_switch_ioloop(conn->log_out); if (cmd_ver2 != NULL) doveadm_cmd_server_run_ver2(conn, argc, argv, cctx);
From 59cd19919bf444e5c3fa429314408aacc8dd4eb8 Mon Sep 17 00:00:00 2001 From: Timo Sirainen <timo.sirai...@dovecot.fi> Date: Tue, 24 Apr 2018 18:47:28 +0300 Subject: [PATCH 1/2] lib-storage: Add mail_user_home_mkdir() --- src/lib-storage/mail-user.c | 61 +++++++++++++++++++++++++++++++++++++++++++++ src/lib-storage/mail-user.h | 5 ++++ 2 files changed, 66 insertions(+) diff --git a/src/lib-storage/mail-user.c b/src/lib-storage/mail-user.c index 947e26cee4..a15ed353ff 100644 --- a/src/lib-storage/mail-user.c +++ b/src/lib-storage/mail-user.c @@ -8,6 +8,7 @@ #include "module-dir.h" #include "home-expand.h" #include "file-create-locked.h" +#include "mkdir-parents.h" #include "safe-mkstemp.h" #include "str.h" #include "strescape.h" @@ -716,6 +717,66 @@ void mail_user_stats_fill(struct mail_user *user, struct stats *stats) user->v.stats_fill(user, stats); } +static int +mail_user_home_mkdir_try_ns(struct mail_namespace *ns, const char *home) +{ + const enum mailbox_list_path_type types[] = { + MAILBOX_LIST_PATH_TYPE_DIR, + MAILBOX_LIST_PATH_TYPE_ALT_DIR, + MAILBOX_LIST_PATH_TYPE_CONTROL, + MAILBOX_LIST_PATH_TYPE_INDEX, + MAILBOX_LIST_PATH_TYPE_INDEX_PRIVATE, + MAILBOX_LIST_PATH_TYPE_INDEX_CACHE, + MAILBOX_LIST_PATH_TYPE_LIST_INDEX, + }; + size_t home_len = strlen(home); + const char *path; + + for (unsigned int i = 0; i < N_ELEMENTS(types); i++) { + if (!mailbox_list_get_root_path(ns->list, types[i], &path)) + continue; + if (strncmp(path, home, home_len) == 0 && + (path[home_len] == '\0' || path[home_len] == '/')) { + return mailbox_list_mkdir_root(ns->list, path, + types[i]) < 0 ? -1 : 1; + } + } + return 0; +} + +int mail_user_home_mkdir(struct mail_user *user) +{ + struct mail_namespace *ns; + const char *home; + int ret; + + if (mail_user_get_home(user, &home) < 0) + return -1; + + /* Try to create the home directory by creating the root directory for + a namespace that exists under the home. This way we end up in the + special mkdir() code in mailbox_list_try_mkdir_root_parent(). + Start from INBOX, since that's usually the correct place. */ + ns = mail_namespace_find_inbox(user->namespaces); + if ((ret = mail_user_home_mkdir_try_ns(ns, home)) != 0) + return ret < 0 ? -1 : 0; + /* try other namespaces */ + for (ns = user->namespaces; ns != NULL; ns = ns->next) { + if ((ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0) { + /* already tried the INBOX namespace */ + continue; + } + if ((ret = mail_user_home_mkdir_try_ns(ns, home)) != 0) + return ret < 0 ? -1 : 0; + } + /* fallback to a safe mkdir() with 0700 mode */ + if (mkdir_parents(home, 0700) < 0 && errno != EEXIST) { + i_error("mkdir_parents(%s) failed: %m", home); + return -1; + } + return 0; +} + static const struct var_expand_func_table mail_user_var_expand_func_table_arr[] = { { "userdb", mail_user_var_expand_func_userdb }, { NULL, NULL } diff --git a/src/lib-storage/mail-user.h b/src/lib-storage/mail-user.h index b97514df75..1627e34fe1 100644 --- a/src/lib-storage/mail-user.h +++ b/src/lib-storage/mail-user.h @@ -210,4 +210,9 @@ void mail_user_init_fs_settings(struct mail_user *user, plugin must be loaded to have anything filled. */ void mail_user_stats_fill(struct mail_user *user, struct stats *stats); +/* Try to mkdir() user's home directory. Ideally this should be called only + after the caller tries to create a file to the home directory, but it fails + with ENOENT. This way it avoids unnecessary disk IO to the home. */ +int mail_user_home_mkdir(struct mail_user *user); + #endif From 8077d714e11388a294f1583e706152396972acce Mon Sep 17 00:00:00 2001 From: Timo Sirainen <timo.sirai...@dovecot.fi> Date: Tue, 24 Apr 2018 18:50:38 +0300 Subject: [PATCH 2/2] dsync: If lock file can't be created because home is missing, mkdir the home --- src/doveadm/dsync/dsync-brain.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/doveadm/dsync/dsync-brain.c b/src/doveadm/dsync/dsync-brain.c index c2b8169f1a..6bc013ff53 100644 --- a/src/doveadm/dsync/dsync-brain.c +++ b/src/doveadm/dsync/dsync-brain.c @@ -441,6 +441,13 @@ dsync_brain_lock(struct dsync_brain *brain, const char *remote_hostname) "/"DSYNC_LOCK_FILENAME, NULL); brain->lock_fd = file_create_locked(brain->lock_path, &lock_set, &brain->lock, &created, &error); + if (brain->lock_fd == -1 && errno == ENOENT) { + /* home directory not created */ + if (mail_user_home_mkdir(brain->user) < 0) + return -1; + brain->lock_fd = file_create_locked(brain->lock_path, &lock_set, + &brain->lock, &created, &error); + } if (brain->lock_fd == -1) i_error("Couldn't lock %s: %s", brain->lock_path, error); else if (brain->debug) {
signature.asc
Description: PGP signature