[Cluster-devel] [PATCH] dlm_controld: trigger network interface failover if a communications error is detected
Support for automatic failover in the face of network interruptions
is being added to the DLM kernel component [1]. This patch aids in that
effort by adding a mechanism whereby userspace can request the DLM
kernel component switch to the next usable network interface.
When --failover is set, dlm_controld will write to a configfs node that
alerts the DLM kernel component to the fact that a communications error
has occurred in userspace. The kernel then reinitializes the DLM
communications stack, binding to the next usable network interface.
The kernel implements a round-robin mechanism for selecting the next
network interface. If necessary, other interface selection heuristics
may be added later.
[1] https://www.redhat.com/archives/cluster-devel/2019-January/msg9.html
Signed-off-by: David Windsor
diff --git a/dlm_controld/action.c b/dlm_controld/action.c
index ecd0d022..c107092d 100644
--- a/dlm_controld/action.c
+++ b/dlm_controld/action.c
@@ -639,6 +639,28 @@ int add_configfs_node(int nodeid, char *addr, int addrlen,
int local)
}
close(fd);
+ /*
+* set failover policy
+*/
+ if opt(failover_ind) {
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "%s/failover", COMMS_DIR);
+
+ fd = open(path, O_WRONLY);
+ if (fd < 0) {
+ log_error("%s: open failed: %d", path, errno);
+ return -1;
+ }
+
+ rv = do_write(fd, (void *)"1", strlen("1"));
+ if (rv < 0) {
+ log_error("%s: write failed: %d", path, errno);
+ close(fd);
+ return -1;
+ }
+ close(fd);
+ }
+
/*
* set local
*/
@@ -681,6 +703,7 @@ int add_configfs_node(int nodeid, char *addr, int addrlen,
int local)
}
close(fd);
}
+
out:
return 0;
}
@@ -907,6 +930,34 @@ int setup_configfs_members(void)
return 0;
}
+/*
+ * Write to the configfs node triggering a switch to the next DLM
+ * failover network interface.
+ */
+int configfs_next_addr(void)
+{
+ int fd, rv;
+ char path[PATH_MAX];
+
+memset(path, 0, PATH_MAX);
+snprintf(path, PATH_MAX, "%s/error", COMMS_DIR);
+
+fd = open(path, O_WRONLY);
+if (fd < 0) {
+log_error("%s: open failed: %d", path, errno);
+return -1;
+}
+
+rv = do_write(fd, (void *)"1", strlen("1"));
+if (rv < 0) {
+log_error("%s: write failed: %d", path, errno);
+close(fd);
+return -1;
+}
+close(fd);
+ return 0;
+}
+
static void find_minors(void)
{
FILE *fl;
diff --git a/dlm_controld/dlm.conf.5 b/dlm_controld/dlm.conf.5
index 09492176..f086dfb1 100644
--- a/dlm_controld/dlm.conf.5
+++ b/dlm_controld/dlm.conf.5
@@ -40,6 +40,8 @@ protocol
.br
bind_all
.br
+failover
+.br
debug_logfile
.br
enable_plock
diff --git a/dlm_controld/dlm_daemon.h b/dlm_controld/dlm_daemon.h
index 3221e19c..9f244fd0 100644
--- a/dlm_controld/dlm_daemon.h
+++ b/dlm_controld/dlm_daemon.h
@@ -96,6 +96,7 @@ enum {
protocol_ind,
debug_logfile_ind,
bind_all_ind,
+ failover_ind,
enable_fscontrol_ind,
enable_plock_ind,
plock_debug_ind,
@@ -363,6 +364,7 @@ void del_configfs_node(int nodeid);
void clear_configfs(void);
int setup_configfs_options(void);
int setup_configfs_members(void);
+int configfs_next_addr(void);
int check_uncontrolled_lockspaces(void);
int setup_misc_devices(void);
int path_exists(const char *path);
diff --git a/dlm_controld/main.c b/dlm_controld/main.c
index 8be6a4bc..ca19eac9 100644
--- a/dlm_controld/main.c
+++ b/dlm_controld/main.c
@@ -1501,6 +1501,7 @@ static int loop(void)
if (pollfd[i].revents & (POLLERR | POLLHUP | POLLNVAL))
{
deadfn = client[i].deadfn;
deadfn(i);
+ configfs_next_addr();
}
}
query_unlock();
@@ -1732,6 +1733,11 @@ static void set_opt_defaults(void)
0, NULL,
""); /* do not advertise */
+ set_opt_default(failover_ind,
+ "failover", '\0', req_arg_int,
+ 0, NULL,
+ ""); /* do not advertise */
+
set_opt_default(debug_logfile_ind,
"debug_logfile", 'L', no_arg,
0, NULL,
@@ -2096,4 +2102,3 @@ int main(int argc, char **argv)
unlink_lockfile(fd, RUNDIR, RUN_FILE_NAME);
return rv < 0 ? 1 : 0;
}
-
--
2.21.0
[Cluster-devel] [PATCH 1/1] dlm_controld: bind to all interfaces for failover
Support for automatic failover in the face of network interruptions
is being added to the DLM kernel component. [1] This patch aids in that
effort by adding a mechanism whereby userspace can convey to the
kernel its intention to use all network addresses for automatic
failover. DLM's current default behavior is to bind to only a single
interface.
When --bind_all is set, dlm_controld will write to a configfs
node that alerts the kernel of its intention to use all local network
addresses for automatic failover. When selecting the next address to
bind to, DLM will iterate through its list of local network addresses
in a round-robin fashion. Support for other address selection
heuritics may be added in the future.
It is important to understand that, per the DLM spec, while DLM
can use a set of addresses for automatic failover, only one address
is considered the active address between two DLM nodes at a time.
This patch does not violate that constraint.
[1] https://www.redhat.com/archives/cluster-devel/2019-January/msg9.html
Signed-off-by: David Windsor
---
dlm_controld/action.c | 19 +++
dlm_controld/dlm.conf.5 | 2 ++
dlm_controld/dlm_daemon.h | 1 +
dlm_controld/main.c | 5 +
4 files changed, 27 insertions(+)
diff --git a/dlm_controld/action.c b/dlm_controld/action.c
index 84637f15..ecd0d022 100644
--- a/dlm_controld/action.c
+++ b/dlm_controld/action.c
@@ -662,6 +662,25 @@ int add_configfs_node(int nodeid, char *addr, int addrlen,
int local)
return -1;
}
close(fd);
+
+ if (opt(bind_all_ind)) {
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "%s/%d/bind_all", COMMS_DIR, nodeid);
+
+ fd = open(path, O_WRONLY);
+ if (fd < 0) {
+ log_error("%s: open failed: %d", path, errno);
+ return -1;
+ }
+
+ rv = do_write(fd, (void *)"1", strlen("1"));
+ if (rv < 0) {
+ log_error("%s: write failed: %d", path, errno);
+ close(fd);
+ return -1;
+ }
+ close(fd);
+ }
out:
return 0;
}
diff --git a/dlm_controld/dlm.conf.5 b/dlm_controld/dlm.conf.5
index 616b60da..09492176 100644
--- a/dlm_controld/dlm.conf.5
+++ b/dlm_controld/dlm.conf.5
@@ -38,6 +38,8 @@ log_debug
.br
protocol
.br
+bind_all
+.br
debug_logfile
.br
enable_plock
diff --git a/dlm_controld/dlm_daemon.h b/dlm_controld/dlm_daemon.h
index 1182c971..3221e19c 100644
--- a/dlm_controld/dlm_daemon.h
+++ b/dlm_controld/dlm_daemon.h
@@ -95,6 +95,7 @@ enum {
timewarn_ind,
protocol_ind,
debug_logfile_ind,
+ bind_all_ind,
enable_fscontrol_ind,
enable_plock_ind,
plock_debug_ind,
diff --git a/dlm_controld/main.c b/dlm_controld/main.c
index 1b60ccda..8be6a4bc 100644
--- a/dlm_controld/main.c
+++ b/dlm_controld/main.c
@@ -1727,6 +1727,11 @@ static void set_opt_defaults(void)
-1, "detect",
"dlm kernel lowcomms protocol: tcp, sctp, detect");
+ set_opt_default(bind_all_ind,
+ "bind_all", '\0', req_arg_int,
+ 0, NULL,
+ ""); /* do not advertise */
+
set_opt_default(debug_logfile_ind,
"debug_logfile", 'L', no_arg,
0, NULL,
--
2.20.1
[Cluster-devel] [PATCH v2 1/3] dlm: check if workqueues are NULL before flushing/destroying
If the DLM lowcomms stack is shut down before any DLM
traffic can be generated, flush_workqueue() and
destroy_workqueue() can be called on empty send and/or recv
workqueues.
Insert guard conditionals to only call flush_workqueue()
and destroy_workqueue() on workqueues that are not NULL.
Signed-off-by: David Windsor
---
fs/dlm/lowcomms.c | 18 --
1 file changed, 12 insertions(+), 6 deletions(-)
diff --git a/fs/dlm/lowcomms.c b/fs/dlm/lowcomms.c
index a5e4a221435c..a93ebffe84b3 100644
--- a/fs/dlm/lowcomms.c
+++ b/fs/dlm/lowcomms.c
@@ -1630,8 +1630,10 @@ static void clean_writequeues(void)
static void work_stop(void)
{
- destroy_workqueue(recv_workqueue);
- destroy_workqueue(send_workqueue);
+ if (recv_workqueue)
+ destroy_workqueue(recv_workqueue);
+ if (send_workqueue)
+ destroy_workqueue(send_workqueue);
}
static int work_start(void)
@@ -1691,13 +1693,17 @@ static void work_flush(void)
struct hlist_node *n;
struct connection *con;
- flush_workqueue(recv_workqueue);
- flush_workqueue(send_workqueue);
+ if (recv_workqueue)
+ flush_workqueue(recv_workqueue);
+ if (send_workqueue)
+ flush_workqueue(send_workqueue);
do {
ok = 1;
foreach_conn(stop_conn);
- flush_workqueue(recv_workqueue);
- flush_workqueue(send_workqueue);
+ if (recv_workqueue)
+ flush_workqueue(recv_workqueue);
+ if (send_workqueue)
+ flush_workqueue(send_workqueue);
for (i = 0; i < CONN_HASH_SIZE && ok; i++) {
hlist_for_each_entry_safe(con, n,
_hash[i], list) {
--
2.20.1
[Cluster-devel] [PATCH v2 2/3] dlm: add TCP multihoming/failover support
Add the ability to specify multiple source addresses
for DLM nodes so that multihomed configurations can
use multiple addresses and still be recognized by the
receiving node.
While each node is capable of being configured for multiple
IPs, DLM requires each node have only one active address
at a time.
This patch introduces a round-robin heuristic for selecting
the next active interface, but other heuristics could
easily be added later.
To support failover, a new configfs node is added by this patch:
/sys/kernel/config/dlm/cluster/comms//error
This node is write-only, and is provided so that userspace
may signal the kernel when it detects a communications error.
The kernel will switch to the next local network interface
after 1 is written to the new configfs node.
Signed-off-by: David Windsor
---
fs/dlm/config.c | 21 +
fs/dlm/lowcomms.c | 60 +--
fs/dlm/lowcomms.h | 1 +
3 files changed, 69 insertions(+), 13 deletions(-)
diff --git a/fs/dlm/config.c b/fs/dlm/config.c
index 1270551d24e3..96db7b1346f9 100644
--- a/fs/dlm/config.c
+++ b/fs/dlm/config.c
@@ -31,6 +31,7 @@
* /config/dlm//comms//local
* /config/dlm//comms//addr (write only)
* /config/dlm//comms//addr_list (read only)
+ * /config/dlm//comms//error(write only)
* The level is useless, but I haven't figured out how to avoid it.
*/
@@ -198,6 +199,7 @@ enum {
COMM_ATTR_LOCAL,
COMM_ATTR_ADDR,
COMM_ATTR_ADDR_LIST,
+ COMM_ATTR_ERROR,
};
enum {
@@ -662,16 +664,35 @@ static ssize_t comm_addr_list_show(struct config_item
*item, char *buf)
return 4096 - allowance;
}
+static ssize_t comm_error_store(struct config_item *item, const char *buf,
+ size_t len)
+{
+ int ret, i;
+
+ ret = kstrtoint(buf, 0, );
+ if (ret < 0)
+ return ret;
+
+ if (i == 0)
+ return 0;
+
+ dlm_lowcomms_next_addr();
+
+ return len;
+}
+
CONFIGFS_ATTR(comm_, nodeid);
CONFIGFS_ATTR(comm_, local);
CONFIGFS_ATTR_WO(comm_, addr);
CONFIGFS_ATTR_RO(comm_, addr_list);
+CONFIGFS_ATTR_WO(comm_, error);
static struct configfs_attribute *comm_attrs[] = {
[COMM_ATTR_NODEID] = _attr_nodeid,
[COMM_ATTR_LOCAL] = _attr_local,
[COMM_ATTR_ADDR] = _attr_addr,
[COMM_ATTR_ADDR_LIST] = _attr_addr_list,
+ [COMM_ATTR_ERROR] = _attr_error,
NULL,
};
diff --git a/fs/dlm/lowcomms.c b/fs/dlm/lowcomms.c
index a93ebffe84b3..be0e134d4fc4 100644
--- a/fs/dlm/lowcomms.c
+++ b/fs/dlm/lowcomms.c
@@ -159,6 +159,8 @@ static DEFINE_SPINLOCK(dlm_node_addrs_spin);
static struct sockaddr_storage *dlm_local_addr[DLM_MAX_ADDR_COUNT];
static int dlm_local_count;
static int dlm_allow_conn;
+static int dlm_local_idx;
+static DEFINE_SPINLOCK(dlm_local_idx_spin);
/* Work queues */
static struct workqueue_struct *recv_workqueue;
@@ -330,7 +332,8 @@ static int nodeid_to_addr(int nodeid, struct
sockaddr_storage *sas_out,
if (!sa_out)
return 0;
- if (dlm_local_addr[0]->ss_family == AF_INET) {
+ spin_lock(_local_idx_spin);
+ if (dlm_local_addr[dlm_local_idx]->ss_family == AF_INET) {
struct sockaddr_in *in4 = (struct sockaddr_in *)
struct sockaddr_in *ret4 = (struct sockaddr_in *) sa_out;
ret4->sin_addr.s_addr = in4->sin_addr.s_addr;
@@ -339,6 +342,7 @@ static int nodeid_to_addr(int nodeid, struct
sockaddr_storage *sas_out,
struct sockaddr_in6 *ret6 = (struct sockaddr_in6 *) sa_out;
ret6->sin6_addr = in6->sin6_addr;
}
+ spin_unlock(_local_idx_spin);
return 0;
}
@@ -519,6 +523,8 @@ static void lowcomms_error_report(struct sock *sk)
dlm_config.ci_tcp_port, sk->sk_err,
sk->sk_err_soft);
}
+
+ dlm_lowcomms_next_addr();
out:
read_unlock_bh(>sk_callback_lock);
if (orig_report)
@@ -572,7 +578,9 @@ static void add_sock(struct socket *sock, struct connection
*con)
static void make_sockaddr(struct sockaddr_storage *saddr, uint16_t port,
int *addr_len)
{
- saddr->ss_family = dlm_local_addr[0]->ss_family;
+ spin_lock(_local_idx_spin);
+ saddr->ss_family = dlm_local_addr[dlm_local_idx]->ss_family;
+ spin_unlock(_local_idx_spin);
if (saddr->ss_family == AF_INET) {
struct sockaddr_in *in4_addr = (struct sockaddr_in *)saddr;
in4_addr->sin_port = cpu_to_be16(port);
@@ -590,8 +598,14 @@ static void make_sockaddr(struct sockaddr_storage *saddr,
uint16_t port,
static void close_connection(struct connection *con, bool and_other,
bool tx, bool rx)
{
- bool closing = test_and_set_bit(CF_CLOSING, >flags);
+ bool closing;
+
+
[Cluster-devel] [PATCH v2 3/3] dlm: allow binding to all network interfaces
Currently, in the kernel, DLM only is able to bind its
listen socket to a single network interface. To support
more robust network configurations, DLM should be able
to bind to all network interfaces.
This patch adds a configfs node to enable/disable binding
to all network interfaces. When 1 is written to this
configfs node, the DLM listen socket will bind to all network
interfaces. When 0 is written to the node, DLM will bind
only to its current local network interface.
Signed-off-by: David Windsor
---
fs/dlm/config.c | 21 +
fs/dlm/config.h | 3 ++-
fs/dlm/lowcomms.c | 19 ++-
3 files changed, 41 insertions(+), 2 deletions(-)
diff --git a/fs/dlm/config.c b/fs/dlm/config.c
index 96db7b1346f9..16b83d61b060 100644
--- a/fs/dlm/config.c
+++ b/fs/dlm/config.c
@@ -29,6 +29,7 @@
* /config/dlm//spaces//nodes//weight
* /config/dlm//comms//nodeid
* /config/dlm//comms//local
+ * /config/dlm//comms//bind_all
* /config/dlm//comms//addr (write only)
* /config/dlm//comms//addr_list (read only)
* /config/dlm//comms//error(write only)
@@ -39,6 +40,7 @@ static struct config_group *space_list;
static struct config_group *comm_list;
static struct dlm_comm *local_comm;
static uint32_t dlm_comm_count;
+static int bind_all;
struct dlm_clusters;
struct dlm_cluster;
@@ -200,6 +202,7 @@ enum {
COMM_ATTR_ADDR,
COMM_ATTR_ADDR_LIST,
COMM_ATTR_ERROR,
+ COMM_ATTR_BIND_ALL,
};
enum {
@@ -681,11 +684,23 @@ static ssize_t comm_error_store(struct config_item *item,
const char *buf,
return len;
}
+static ssize_t comm_bind_all_show(struct config_item *item, char *buf)
+{
+ return sprintf(buf, "%d\n", bind_all);
+}
+
+static ssize_t comm_bind_all_store(struct config_item *item, const char *buf,
+ size_t len)
+{
+ return kstrtoint(buf, 0, _all);
+}
+
CONFIGFS_ATTR(comm_, nodeid);
CONFIGFS_ATTR(comm_, local);
CONFIGFS_ATTR_WO(comm_, addr);
CONFIGFS_ATTR_RO(comm_, addr_list);
CONFIGFS_ATTR_WO(comm_, error);
+CONFIGFS_ATTR(comm_, bind_all);
static struct configfs_attribute *comm_attrs[] = {
[COMM_ATTR_NODEID] = _attr_nodeid,
@@ -693,6 +708,7 @@ static struct configfs_attribute *comm_attrs[] = {
[COMM_ATTR_ADDR] = _attr_addr,
[COMM_ATTR_ADDR_LIST] = _attr_addr_list,
[COMM_ATTR_ERROR] = _attr_error,
+ [COMM_ATTR_BIND_ALL] = _attr_bind_all,
NULL,
};
@@ -868,6 +884,11 @@ int dlm_our_addr(struct sockaddr_storage *addr, int num)
return 0;
}
+int dlm_bind_all(void)
+{
+ return bind_all;
+}
+
/* Config file defaults */
#define DEFAULT_TCP_PORT 21064
#define DEFAULT_BUFFER_SIZE 4096
diff --git a/fs/dlm/config.h b/fs/dlm/config.h
index 6041eec886ab..e3fd8ce45874 100644
--- a/fs/dlm/config.h
+++ b/fs/dlm/config.h
@@ -21,7 +21,7 @@ struct dlm_config_node {
uint32_t comm_seq;
};
-#define DLM_MAX_ADDR_COUNT 3
+#define DLM_MAX_ADDR_COUNT 9
struct dlm_config_info {
int ci_tcp_port;
@@ -49,6 +49,7 @@ int dlm_config_nodes(char *lsname, struct dlm_config_node
**nodes_out,
int dlm_comm_seq(int nodeid, uint32_t *seq);
int dlm_our_nodeid(void);
int dlm_our_addr(struct sockaddr_storage *addr, int num);
+int dlm_bind_all(void);
#endif /* __CONFIG_DOT_H__ */
diff --git a/fs/dlm/lowcomms.c b/fs/dlm/lowcomms.c
index be0e134d4fc4..60ae7c53a8a1 100644
--- a/fs/dlm/lowcomms.c
+++ b/fs/dlm/lowcomms.c
@@ -1394,6 +1394,9 @@ static int sctp_listen_for_all(void)
static int tcp_listen_for_all(void)
{
struct socket *sock = NULL;
+ struct sockaddr_in *sin4;
+ struct sockaddr_in6 *sin6;
+ struct sockaddr_storage sas, laddr;
struct connection *con = nodeid2con(0, GFP_NOFS);
int result = -EINVAL;
@@ -1402,7 +1405,21 @@ static int tcp_listen_for_all(void)
log_print("Using TCP for communications");
- sock = tcp_create_listen_sock(con, dlm_local_addr[dlm_local_idx]);
+ memcpy(, dlm_local_addr[dlm_local_idx], sizeof(sas));
+ memcpy(, dlm_local_addr[dlm_local_idx], sizeof(laddr));
+ if (dlm_bind_all()) {
+ if (sas.ss_family == AF_INET) {
+ sin4 = (struct sockaddr_in *)
+ sin4->sin_addr.s_addr = htonl(INADDR_ANY);
+ memcpy(, sin4, sizeof(laddr));
+ } else {
+ sin6 = (struct sockaddr_in6 *)
+ sin6->sin6_addr = in6addr_any;
+ memcpy(, sin6, sizeof(laddr));
+ }
+ }
+
+ sock = tcp_create_listen_sock(con, );
if (sock) {
add_sock(sock, con);
result = 0;
--
2.20.1
[Cluster-devel] [PATCH 1/3] dlm: check if workqueues are NULL before destroying
If a network failure occurs before any DLM traffic can be
generated, the send and receive workqueues can be NULL
when work_stop() is called. Check to see if these workqueues
are NULL before calling destroy_workqueue().
Signed-off-by: David Windsor
---
fs/dlm/lowcomms.c | 6 --
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/fs/dlm/lowcomms.c b/fs/dlm/lowcomms.c
index 76976d6e50f9..905cbdbd31bc 100644
--- a/fs/dlm/lowcomms.c
+++ b/fs/dlm/lowcomms.c
@@ -1630,8 +1630,10 @@ static void clean_writequeues(void)
static void work_stop(void)
{
- destroy_workqueue(recv_workqueue);
- destroy_workqueue(send_workqueue);
+ if (recv_workqueue)
+ destroy_workqueue(recv_workqueue);
+ if (send_workqueue)
+ destroy_workqueue(send_workqueue);
}
static int work_start(void)
--
2.20.1
[Cluster-devel] [PATCH 2/3] dlm: add TCP multihoming/failover support
Add the ability to specify multiple source addresses
for DLM nodes so that multihomed configurations can
use multiple addresses and still be recognized by the
receiving node.
While each node is capable of being configured for multiple
IPs, DLM requires each node have only one active address
at a time.
This patch introduces a round-robin heuristic for selecting
the next active interface, but other heuristics could
easily be added later.
To support failover, a new configfs node is added by this patch:
/sys/kernel/config/dlm/cluster/comms//error
This node is write-only, and is provided so that userspace
may signal the kernel when it detects a communications error.
The kernel will switch to the next local network interface
after 1 is written to the new configfs node.
Signed-off-by: David Windsor
---
fs/dlm/config.c | 21 +
fs/dlm/lowcomms.c | 34 +++---
fs/dlm/lowcomms.h | 1 +
3 files changed, 45 insertions(+), 11 deletions(-)
diff --git a/fs/dlm/config.c b/fs/dlm/config.c
index 1270551d24e3..23d2677e10bd 100644
--- a/fs/dlm/config.c
+++ b/fs/dlm/config.c
@@ -31,6 +31,7 @@
* /config/dlm//comms//local
* /config/dlm//comms//addr (write only)
* /config/dlm//comms//addr_list (read only)
+ * /config/dlm//comms//error(write only)
* The level is useless, but I haven't figured out how to avoid it.
*/
@@ -198,6 +199,7 @@ enum {
COMM_ATTR_LOCAL,
COMM_ATTR_ADDR,
COMM_ATTR_ADDR_LIST,
+ COMM_ATTR_ERROR,
};
enum {
@@ -662,8 +664,26 @@ static ssize_t comm_addr_list_show(struct config_item
*item, char *buf)
return 4096 - allowance;
}
+static ssize_t comm_error_store(struct config_item *item, const char *buf,
+ size_t len)
+{
+ int ret, i;
+
+ ret = kstrtoint(buf, 0, );
+ if (ret < 0)
+ return ret;
+
+ if (i == 0)
+ return 0;
+
+ dlm_lowcomms_next_addr();
+
+ return len;
+}
+
CONFIGFS_ATTR(comm_, nodeid);
CONFIGFS_ATTR(comm_, local);
+CONFIGFS_ATTR_WO(comm_, error);
CONFIGFS_ATTR_WO(comm_, addr);
CONFIGFS_ATTR_RO(comm_, addr_list);
@@ -672,6 +692,7 @@ static struct configfs_attribute *comm_attrs[] = {
[COMM_ATTR_LOCAL] = _attr_local,
[COMM_ATTR_ADDR] = _attr_addr,
[COMM_ATTR_ADDR_LIST] = _attr_addr_list,
+ [COMM_ATTR_ERROR] = _attr_error,
NULL,
};
diff --git a/fs/dlm/lowcomms.c b/fs/dlm/lowcomms.c
index 905cbdbd31bc..d37af1372ed0 100644
--- a/fs/dlm/lowcomms.c
+++ b/fs/dlm/lowcomms.c
@@ -159,6 +159,7 @@ static DEFINE_SPINLOCK(dlm_node_addrs_spin);
static struct sockaddr_storage *dlm_local_addr[DLM_MAX_ADDR_COUNT];
static int dlm_local_count;
static int dlm_allow_conn;
+static int dlm_local_idx;
/* Work queues */
static struct workqueue_struct *recv_workqueue;
@@ -330,7 +331,7 @@ static int nodeid_to_addr(int nodeid, struct
sockaddr_storage *sas_out,
if (!sa_out)
return 0;
- if (dlm_local_addr[0]->ss_family == AF_INET) {
+ if (dlm_local_addr[dlm_local_idx]->ss_family == AF_INET) {
struct sockaddr_in *in4 = (struct sockaddr_in *)
struct sockaddr_in *ret4 = (struct sockaddr_in *) sa_out;
ret4->sin_addr.s_addr = in4->sin_addr.s_addr;
@@ -519,6 +520,8 @@ static void lowcomms_error_report(struct sock *sk)
dlm_config.ci_tcp_port, sk->sk_err,
sk->sk_err_soft);
}
+
+ dlm_lowcomms_next_addr();
out:
read_unlock_bh(>sk_callback_lock);
if (orig_report)
@@ -572,7 +575,7 @@ static void add_sock(struct socket *sock, struct connection
*con)
static void make_sockaddr(struct sockaddr_storage *saddr, uint16_t port,
int *addr_len)
{
- saddr->ss_family = dlm_local_addr[0]->ss_family;
+ saddr->ss_family = dlm_local_addr[dlm_local_idx]->ss_family;
if (saddr->ss_family == AF_INET) {
struct sockaddr_in *in4_addr = (struct sockaddr_in *)saddr;
in4_addr->sin_port = cpu_to_be16(port);
@@ -1169,7 +1172,7 @@ static void tcp_connect_to_sock(struct connection *con)
/* Bind to our cluster-known address connecting to avoid
routing problems */
- memcpy(_addr, dlm_local_addr[0], sizeof(src_addr));
+ memcpy(_addr, dlm_local_addr[dlm_local_idx], sizeof(src_addr));
make_sockaddr(_addr, 0, _len);
result = sock->ops->bind(sock, (struct sockaddr *) _addr,
addr_len);
@@ -1213,6 +1216,7 @@ static void tcp_connect_to_sock(struct connection *con)
con->retries, result);
mutex_unlock(>sock_mutex);
msleep(1000);
+ dlm_lowcomms_next_addr();
lowcomms_connect_sock(con);
return;
}
@@ -1293,6
[Cluster-devel] [PATCH 3/3] dlm: allow binding to all network interfaces
Currently, in the kernel, DLM only is able to bind its
listen socket to a single network interface. To support
more robust network configurations, DLM should be able
to bind to all network interfaces.
This patch adds a configfs node to enable/disable binding
to all network interfaces. When 1 is written to this
configfs node, the DLM listen socket will bind to all network
interfaces. When 0 is written to the node, DLM will bind
only to its current local network interface.
Signed-off-by: David Windsor
---
fs/dlm/config.c | 21 +
fs/dlm/config.h | 3 ++-
fs/dlm/lowcomms.c | 19 ++-
3 files changed, 41 insertions(+), 2 deletions(-)
diff --git a/fs/dlm/config.c b/fs/dlm/config.c
index 23d2677e10bd..77dc325c1972 100644
--- a/fs/dlm/config.c
+++ b/fs/dlm/config.c
@@ -29,6 +29,7 @@
* /config/dlm//spaces//nodes//weight
* /config/dlm//comms//nodeid
* /config/dlm//comms//local
+ * /config/dlm//comms//bind_all
* /config/dlm//comms//addr (write only)
* /config/dlm//comms//addr_list (read only)
* /config/dlm//comms//error(write only)
@@ -39,6 +40,7 @@ static struct config_group *space_list;
static struct config_group *comm_list;
static struct dlm_comm *local_comm;
static uint32_t dlm_comm_count;
+static int bind_all;
struct dlm_clusters;
struct dlm_cluster;
@@ -197,6 +199,7 @@ static struct configfs_attribute *cluster_attrs[] = {
enum {
COMM_ATTR_NODEID = 0,
COMM_ATTR_LOCAL,
+ COMM_ATTR_BIND_ALL,
COMM_ATTR_ADDR,
COMM_ATTR_ADDR_LIST,
COMM_ATTR_ERROR,
@@ -681,8 +684,20 @@ static ssize_t comm_error_store(struct config_item *item,
const char *buf,
return len;
}
+static ssize_t comm_bind_all_show(struct config_item *item, char *buf)
+{
+ return sprintf(buf, "%d\n", bind_all);
+}
+
+static ssize_t comm_bind_all_store(struct config_item *item, const char *buf,
+ size_t len)
+{
+ return kstrtoint(buf, 0, _all);
+}
+
CONFIGFS_ATTR(comm_, nodeid);
CONFIGFS_ATTR(comm_, local);
+CONFIGFS_ATTR(comm_, bind_all);
CONFIGFS_ATTR_WO(comm_, error);
CONFIGFS_ATTR_WO(comm_, addr);
CONFIGFS_ATTR_RO(comm_, addr_list);
@@ -693,6 +708,7 @@ static struct configfs_attribute *comm_attrs[] = {
[COMM_ATTR_ADDR] = _attr_addr,
[COMM_ATTR_ADDR_LIST] = _attr_addr_list,
[COMM_ATTR_ERROR] = _attr_error,
+ [COMM_ATTR_BIND_ALL] = _attr_bind_all,
NULL,
};
@@ -868,6 +884,11 @@ int dlm_our_addr(struct sockaddr_storage *addr, int num)
return 0;
}
+int dlm_bind_all(void)
+{
+ return bind_all;
+}
+
/* Config file defaults */
#define DEFAULT_TCP_PORT 21064
#define DEFAULT_BUFFER_SIZE 4096
diff --git a/fs/dlm/config.h b/fs/dlm/config.h
index 6041eec886ab..e3fd8ce45874 100644
--- a/fs/dlm/config.h
+++ b/fs/dlm/config.h
@@ -21,7 +21,7 @@ struct dlm_config_node {
uint32_t comm_seq;
};
-#define DLM_MAX_ADDR_COUNT 3
+#define DLM_MAX_ADDR_COUNT 9
struct dlm_config_info {
int ci_tcp_port;
@@ -49,6 +49,7 @@ int dlm_config_nodes(char *lsname, struct dlm_config_node
**nodes_out,
int dlm_comm_seq(int nodeid, uint32_t *seq);
int dlm_our_nodeid(void);
int dlm_our_addr(struct sockaddr_storage *addr, int num);
+int dlm_bind_all(void);
#endif /* __CONFIG_DOT_H__ */
diff --git a/fs/dlm/lowcomms.c b/fs/dlm/lowcomms.c
index d37af1372ed0..8b9d02485116 100644
--- a/fs/dlm/lowcomms.c
+++ b/fs/dlm/lowcomms.c
@@ -1374,6 +1374,9 @@ static int sctp_listen_for_all(void)
static int tcp_listen_for_all(void)
{
struct socket *sock = NULL;
+ struct sockaddr_in *sin4;
+ struct sockaddr_in6 *sin6;
+ struct sockaddr_storage sas, laddr;
struct connection *con = nodeid2con(0, GFP_NOFS);
int result = -EINVAL;
@@ -1382,7 +1385,21 @@ static int tcp_listen_for_all(void)
log_print("Using TCP for communications");
- sock = tcp_create_listen_sock(con, dlm_local_addr[dlm_local_idx]);
+ memcpy(, dlm_local_addr[dlm_local_idx], sizeof(sas));
+ memcpy(, dlm_local_addr[dlm_local_idx], sizeof(laddr));
+ if (dlm_bind_all()) {
+ if (sas.ss_family == AF_INET) {
+ sin4 = (struct sockaddr_in *)
+ sin4->sin_addr.s_addr = htonl(INADDR_ANY);
+ memcpy(, sin4, sizeof(laddr));
+ } else {
+ sin6 = (struct sockaddr_in6 *)
+ sin6->sin6_addr = in6addr_any;
+ memcpy(, sin6, sizeof(laddr));
+ }
+ }
+
+ sock = tcp_create_listen_sock(con, );
if (sock) {
add_sock(sock, con);
result = 0;
--
2.20.1
