16.01.2012 09:20, Andrew Beekhof wrote:
[snip]
>>> At the same time, stonith_admin -B succeeds.
>>> The main difference I see is st_opt_sync_call in a latter case.
>>> Will try to experiment with it.
>>
>> Yeeeesssss!!!
>>
>> Now I see following:
>> Dec 19 11:53:34 vd01-a cluster-dlm: [2474]: info:
>> pacemaker_terminate_member: Requesting that node 1090782474/vd01-b be fenced
>
> So the important question... what did you change?
Nice you're back ;)
+ rc = st->cmds->fence(st, *st_opt_sync_call*, node_uname, "reboot", 120);
attaching my resulting version of pacemaker.c (which still has a lot of
mess because of different approaches I tried to get the result and needs
a cleanup). Function you may look at is pacemaker_terminate_member()
which is almost one-to-one copy of crm_terminate_member_no_mainloop()
except rename of variable to compile without warnings and change of
->fence() arguments.
>
>> Dec 19 11:53:34 vd01-a stonith-ng: [1905]: info:
>> initiate_remote_stonith_op: Initiating remote operation reboot for
>> vd01-b: 21425fc0-4311-40fa-9647-525c3f258471
>> Dec 19 11:53:34 vd01-a stonith-ng: [1905]: info: crm_get_peer: Node
>> vd01-c now has id: 1107559690
>> Dec 19 11:53:34 vd01-a stonith-ng: [1905]: info: stonith_command:
>> Processed st_query from vd01-c: rc=0
>> Dec 19 11:53:34 vd01-a stonith-ng: [1905]: info: crm_get_peer: Node
>> vd01-d now has id: 1124336906
>> Dec 19 11:53:34 vd01-a stonith-ng: [1905]: info: stonith_command:
>> Processed st_query from vd01-d: rc=0
>> Dec 19 11:53:34 vd01-a stonith-ng: [1905]: info: stonith_command:
>> Processed st_query from vd01-a: rc=0
>> Dec 19 11:53:34 vd01-a stonith-ng: [1905]: info: call_remote_stonith:
>> Requesting that vd01-c perform op reboot vd01-b
>> Dec 19 11:53:34 vd01-a stonith-ng: [1905]: info: crm_get_peer: Node
>> vd01-b now has id: 1090782474
>> ...
>> Dec 19 11:53:40 vd01-a stonith-ng: [1905]: info: stonith_command:
>> Processed st_fence_history from cluster-dlm: rc=0
>> Dec 19 11:53:40 vd01-a crmd: [1910]: info: tengine_stonith_notify: Peer
>> vd01-b was terminated (reboot) by vd01-c for vd01-a
>> (ref=21425fc0-4311-40fa-9647-525c3f258471): OK
>>
>> But, then I see minor issue that node is marked to be fenced again:
>> Dec 19 11:53:40 vd01-a pengine: [1909]: WARN: pe_fence_node: Node vd01-b
>> will be fenced because it is un-expectedly down
>
> Do you have logs for that?
> tengine_stonith_notify() got called, that should have been enough to
> get the node cleaned up in the cib.
Ugh, seems like yes, but they are archived already. Will get them back
to nodes and try to compose hb_report for them (but pe inputs are
already lost, do you still need logs without them?)
>
>> ...
>> Dec 19 11:53:40 vd01-a pengine: [1909]: WARN: stage6: Scheduling Node
>> vd01-b for STONITH
>> ...
>> Dec 19 11:53:40 vd01-a crmd: [1910]: info: te_fence_node: Executing
>> reboot fencing operation (249) on vd01-b (timeout=60000)
>> ...
>> Dec 19 11:53:40 vd01-a stonith-ng: [1905]: info: call_remote_stonith:
>> Requesting that vd01-c perform op reboot vd01-b
>>
>> And so on.
>>
>> I can't investigated this one in more depth, because I use fence_xvm in
>> this testing cluster, and it has issues when running more than one
>> stonith resource on a node. Also, my RA (in a cluster where this testing
>> cluster runs) undefines VM after failure, so fence_xvm does not see
>> fencing victim in a qpid and is unable to fence it again.
>>
>> May be it is possible to look if node was just fenced and skip redundant
>> fencing?
>
> If the callbacks are being used correctly, it shouldn't be required
#include <syslog.h>
#include "config.h"
#include "dlm_daemon.h"
#include <glib.h>
#include <bzlib.h>
#include <heartbeat/ha_msg.h>
#include <pacemaker/crm_config.h>
#include <pacemaker/crm/crm.h>
#include <pacemaker/crm/ais.h>
#include <pacemaker/crm/attrd.h>
/* heartbeat support is irrelevant here */
#undef SUPPORT_HEARTBEAT
#define SUPPORT_HEARTBEAT 0
#include <pacemaker/crm/common/cluster.h>
#include <pacemaker/crm/cluster/stack.h>
#include <pacemaker/crm/common/ipc.h>
#include <pacemaker/crm/msg_xml.h>
#include <pacemaker/crm/cib.h>
#include <pacemaker/crm/stonith-ng.h>
#define COMMS_DIR "/sys/kernel/config/dlm/cluster/comms"
int setup_ccs(void)
{
/* To avoid creating an additional place for the dlm to be configured,
* only allow configuration from the command-line until CoroSync is stable
* enough to be used with Pacemaker
*/
cfgd_groupd_compat = 0; /* always use libcpg and disable backward compat */
return 0;
}
void close_ccs(void) { return; }
int get_weight(int nodeid, char *lockspace) { return 1; }
/* TODO: Make this configurable
* Can't use logging.c as-is as whitetank exposes a different logging API
*/
void init_logging(void) {
openlog("cluster-dlm", LOG_PERROR|LOG_PID|LOG_CONS|LOG_NDELAY, LOG_DAEMON);
/* cl_log_enable_stderr(TRUE); */
}
void setup_logging(void) { return; }
void close_logging(void) {
closelog();
}
extern int ais_fd_async;
char *local_node_uname = NULL;
void dlm_process_node(gpointer key, gpointer value, gpointer user_data);
int setup_cluster(void)
{
ais_fd_async = -1;
crm_log_init("cluster-dlm", LOG_INFO, FALSE, TRUE, 0, NULL);
if(init_ais_connection(NULL, NULL, NULL, &local_node_uname, &our_nodeid) == FALSE) {
log_error("Connection to our AIS plugin failed");
return -1;
}
/* Sign up for membership updates */
send_ais_text(crm_class_notify, "true", TRUE, NULL, crm_msg_ais);
/* Requesting the current list of known nodes */
send_ais_text(crm_class_members, __FUNCTION__, TRUE, NULL, crm_msg_ais);
return ais_fd_async;
}
void update_cluster(void)
{
static uint64_t last_membership = 0;
cluster_quorate = crm_have_quorum;
if(last_membership < crm_peer_seq) {
log_debug("Processing membership %llu", crm_peer_seq);
g_hash_table_foreach(crm_peer_id_cache, dlm_process_node, &last_membership);
last_membership = crm_peer_seq;
}
}
void process_cluster(int ci)
{
ais_dispatch(ais_fd_async, NULL);
update_cluster();
}
void close_cluster(void) {
terminate_ais_connection();
}
#include <arpa/inet.h>
#include <corosync/totem/totemip.h>
void dlm_process_node(gpointer key, gpointer value, gpointer user_data)
{
int rc = 0;
struct stat tmp;
char path[PATH_MAX];
crm_node_t *node = value;
uint64_t *last = user_data;
const char *action = "Skipped";
gboolean do_add = FALSE;
gboolean do_remove = FALSE;
gboolean is_active = FALSE;
memset(path, 0, PATH_MAX);
snprintf(path, PATH_MAX, "%s/%d", COMMS_DIR, node->id);
rc = stat(path, &tmp);
is_active = crm_is_member_active(node);
if(rc == 0 && is_active) {
/* nothing to do?
* maybe the node left and came back...
*/
} else if(rc == 0) {
do_remove = TRUE;
} else if(is_active && node->addr) {
do_add = TRUE;
}
if(do_remove) {
action = "Removed";
del_configfs_node(node->id);
}
if(do_add) {
char *addr_copy = strdup(node->addr);
char *addr_top = addr_copy;
char *addr = NULL;
if(do_remove) {
action = "Re-added";
} else {
action = "Added";
}
do {
char ipaddr[1024];
int addr_family = AF_INET;
int cna_len = 0;
struct sockaddr_storage cna_addr;
struct totem_ip_address totem_addr;
addr = strsep(&addr_copy, " ");
if(addr == NULL) {
break;
}
/* do_cmd_get_node_addrs */
if(strstr(addr, "ip(") == NULL) {
continue;
} else if(strchr(addr, ':')) {
rc = sscanf(addr, "ip(%[0-9A-Fa-f:])", ipaddr);
if(rc != 1) {
log_error("Could not extract IPv6 address from '%s'", addr);
continue;
}
addr_family = AF_INET6;
} else {
rc = sscanf(addr, "ip(%[0-9.]) ", ipaddr);
if(rc != 1) {
log_error("Could not extract IPv4 address from '%s'", addr);
continue;
}
}
rc = inet_pton(addr_family, ipaddr, &totem_addr);
if(rc != 1) {
log_error("Could not parse '%s' as in IPv%c address", ipaddr, (addr_family==AF_INET)?'4':'6');
continue;
}
rc = totemip_parse(&totem_addr, ipaddr, addr_family);
if(rc != 0) {
log_error("Could not convert '%s' into a totem address", ipaddr);
continue;
}
rc = totemip_totemip_to_sockaddr_convert(&totem_addr, 0, &cna_addr, &cna_len);
if(rc != 0) {
log_error("Could not convert totem address for '%s' into sockaddr", ipaddr);
continue;
}
log_debug("Adding address %s to configfs for node %u", addr, node->id);
add_configfs_node(node->id, ((char*)&cna_addr), cna_len, (node->id == our_nodeid));
} while(addr != NULL);
free(addr_top);
}
log_debug("%s %sctive node %u: born-on=%llu, last-seen=%llu, this-event=%llu, last-event=%llu",
action, crm_is_member_active(value)?"a":"ina",
node->id, node->born, node->last_seen,
crm_peer_seq, (unsigned long long)*last);
}
int is_cluster_member(int nodeid)
{
crm_node_t *node = crm_get_peer(nodeid, NULL);
return crm_is_member_active(node);
}
char *nodeid2name(int nodeid) {
crm_node_t *node = crm_get_peer(nodeid, NULL);
if(node->uname == NULL) {
return NULL;
}
return strdup(node->uname);
}
static int
pacemaker_terminate_member(int nodeid)
{
int rc = stonith_ok;
const char *node_uname = NULL;
stonith_t *st = NULL;
if (node_uname == NULL) {
crm_node_t *node = crm_get_peer(nodeid, node_uname);
if (node) {
node_uname = node->uname;
}
}
if (node_uname == NULL) {
crm_err("Nothing known about node id=%d", nodeid);
return -1;
} else {
st = stonith_api_new();
}
if (st) {
rc = st->cmds->connect(st, crm_system_name, NULL);
} else {
crm_err("Could not connect to stonith subsystem");
return -1;
}
if (rc == stonith_ok) {
/* Default pacemaker fencing action is "reboot", and admin may remap it to "off" */
crm_info("Requesting that node %d/%s be fenced", nodeid, node_uname);
rc = st->cmds->fence(st, st_opt_sync_call, node_uname, "reboot", 120);
}
st->cmds->disconnect(st);
stonith_api_delete(st);
if (rc < stonith_ok) {
crm_err("Could not fence node %d/%s: %s", nodeid, crm_str(node_uname), stonith_error2string(rc));
rc = 1;
} else {
rc = 0;
}
return rc;
}
void kick_node_from_cluster(int nodeid)
{
int rc = pacemaker_terminate_member(nodeid);
switch(rc) {
case 0:
log_debug("Requested that node %d be kicked from the cluster", nodeid);
break;
case -1:
log_error("Don't know how to kick node %d from the cluster", nodeid);
break;
case 1:
log_error("Could not kick node %d from the cluster", nodeid);
break;
default:
log_error("Unknown result when kicking node %d from the cluster", nodeid);
break;
}
return;
}
int fence_in_progress(int *in_progress)
{
*in_progress = 0;
return 0;
}
int fence_node_time(int nodeid, uint64_t *last_fenced_time)
{
int rc = 0;
const char *node_uname = NULL;
crm_node_t *node = crm_get_peer(nodeid, node_uname);
stonith_history_t *history, *hp = NULL;
stonith_t *st = NULL;
int fencing_requested = 0;
int refence = 0;
if(last_fenced_time) {
*last_fenced_time = 0;
}
if (node && node->uname) {
node_uname = node->uname;
st = stonith_api_new();
} else {
crm_err("Nothing known about node id=%d", nodeid);
return 0;
}
if(st) {
rc = st->cmds->connect(st, crm_system_name, NULL);
}
if(rc == stonith_ok) {
int i = 0;
st->cmds->history(st, st_opt_sync_call, node_uname, &history, 120);
for(hp = history; hp; hp = hp->next, i++) {
if(hp->state == st_done) {
log_debug("Stonith history[%d]: Node %d/%s fenced at %u", i, nodeid, node_uname, hp->completed);
*last_fenced_time = hp->completed;
fencing_requested = 0;
}else if (hp->state == st_failed) {
log_debug("Stonith history[%d]: Fencing of node %d/%s failed, reqesting it again", i, nodeid, node_uname);
kick_node_from_cluster(nodeid);
refence = 1;
fencing_requested = 0;
} else if (hp->state == st_exec) {
log_debug("Stonith history[%d]: Fencing of node %d/%s is in progress", i, nodeid, node_uname);
fencing_requested = 1;
rc = 2;
} else {
log_debug("Stonith history[%d]: Node %d/%s state %d at %d", i, nodeid, node_uname, hp->state, hp->completed);
}
}
}
rc = 0;
if(*last_fenced_time != 0) {
log_debug("Node %d/%s was last shot at: %d", nodeid, node_uname, *last_fenced_time);
} else {
if(!fencing_requested && !refence) {
log_debug("It does not appear node %d/%s has been shot", nodeid, node_uname);
rc = 1;
}
}
if(st) {
st->cmds->disconnect(st);
stonith_api_delete(st);
}
return rc;
}
_______________________________________________
Pacemaker mailing list: Pacemaker@oss.clusterlabs.org
http://oss.clusterlabs.org/mailman/listinfo/pacemaker
Project Home: http://www.clusterlabs.org
Getting started: http://www.clusterlabs.org/doc/Cluster_from_Scratch.pdf
Bugs: http://bugs.clusterlabs.org
