I did few small changes and fix a problem in function register_dev. --- configure.ac | 8 + fence/agents/scsi/Makefile.am | 20 +- fence/agents/scsi/fence_scsi.8 | 119 ----- fence/agents/scsi/fence_scsi.pl | 875 ---------------------------------- fence/agents/scsi/fence_scsi.py | 452 ++++++++++++++++++ fence/agents/scsi/fence_scsi_check.pl | 170 ------- make/fencebuild.mk | 4 + 7 files changed, 475 insertions(+), 1173 deletions(-) delete mode 100644 fence/agents/scsi/fence_scsi.8 delete mode 100644 fence/agents/scsi/fence_scsi.pl create mode 100644 fence/agents/scsi/fence_scsi.py delete mode 100644 fence/agents/scsi/fence_scsi_check.pl
diff --git a/configure.ac b/configure.ac index 5e37ee5..f208bef 100644 --- a/configure.ac +++ b/configure.ac @@ -163,6 +163,10 @@ CLUSTERDATA=${datadir}/cluster AC_PATH_PROG([IPMITOOL_PATH], [ipmitool], [/usr/bin/ipmitool]) AC_PATH_PROG([AMTTOOL_PATH], [amttool], [/usr/bin/amttool]) AC_PATH_PROG([GNUTLSCLI_PATH], [gnutlscli], [/usr/bin/gnutls-cli]) +AC_PATH_PROG([COROSYNC_CMAPCTL_PATH], [corosync-cmapctl], [/usr/sbin/corosync-cmapctl]) +AC_PATH_PROG([SG_PERSIST_PATH], [sg_persist], [/usr/bin/sg_persist]) +AC_PATH_PROG([SG_TURS_PATH], [sg_turs], [/usr/bin/sg_turs]) +AC_PATH_PROG([VGS_PATH], [vgs], [/usr/sbin/vgs]) ## do subst AC_SUBST([DEFAULT_CONFIG_DIR]) @@ -189,6 +193,10 @@ AM_CONDITIONAL(BUILD_XENAPILIB, test $XENAPILIB -eq 1) AC_SUBST([IPMITOOL_PATH]) AC_SUBST([AMTTOOL_PATH]) +AC_SUBST([COROSYNC_CMAPCTL_PATH]) +AC_SUBST([SG_PERSIST_PATH]) +AC_SUBST([SG_TURS_PATH]) +AC_SUBST([VGS_PATH]) ## *FLAGS handling diff --git a/fence/agents/scsi/Makefile.am b/fence/agents/scsi/Makefile.am index 5652bda..896406a 100644 --- a/fence/agents/scsi/Makefile.am +++ b/fence/agents/scsi/Makefile.am @@ -2,20 +2,22 @@ MAINTAINERCLEANFILES = Makefile.in TARGET = fence_scsi -SRC = $(TARGET).pl +SYMTARGET = fence_scsi_check -EXTRA_DIST = $(SRC) \ - $(TARGET)_check.pl +SRC = $(TARGET).py -scsidatadir = $(CLUSTERDATA) +EXTRA_DIST = $(SRC) -scsidata_SCRIPTS = $(TARGET)_check.pl +sbin_SCRIPTS = $(TARGET) $(SYMTARGET) -sbin_SCRIPTS = $(TARGET) +man_MANS = $(TARGET).8 -dist_man_MANS = $(TARGET).8 +$(SYMTARGET): $(TARGET) + ln -s $^ $@ include $(top_srcdir)/make/fencebuild.mk +include $(top_srcdir)/make/fenceman.mk +include $(top_srcdir)/make/agentpycheck.mk -clean-local: - rm -f $(TARGET) +clean-local: clean-man + rm -f $(TARGET) $(SYMTARGET) diff --git a/fence/agents/scsi/fence_scsi.8 b/fence/agents/scsi/fence_scsi.8 deleted file mode 100644 index 180de4f..0000000 --- a/fence/agents/scsi/fence_scsi.8 +++ /dev/null @@ -1,119 +0,0 @@ -.TH fence_scsi 8 - -.SH NAME -fence_scsi - I/O fencing agent for SCSI persistent reservations - -.SH SYNOPSIS -.B -fence_scsi -[\fIOPTION\fR]... - -.SH DESCRIPTION -fence_scsi is an I/O fencing agent that uses SCSI-3 persistent -reservations to control access to shared storage devices. These -devices must support SCSI-3 persistent reservations (SPC-3 or greater) -as well as the "preempt-and-abort" subcommand. - -The fence_scsi agent works by having each node in the cluster register -a unique key with the SCSI devive(s). Once registered, a single node -will become the reservation holder by creating a "write exclusive, -registrants only" reservation on the device(s). The result is that -only registered nodes may write to the device(s). When a node failure -occurs, the fence_scsi agent will remove the key belonging to the -failed node from the device(s). The failed node will no longer be able -to write to the device(s). A manual reboot is required. In the cluster -environment unfence action should be configured also. - -Keys are either be specified manually (see -k option) or generated -automatically (see -n option). Automatic key generation requires that -cman be running. Keys will then be generated using the cluster ID and -node ID such that each node has a unique key that can be determined by -any other node in the cluster. - -Devices can either be specified manually (see -d option) or discovered -automatically. Multiple devices can be specified manually by using a -comma-separated list. If no devices are specified, the fence_scsi -agent will attempt to discover devices by looking for cluster volumes -and extracting the underlying devices. Devices may be device-mapper -multipath devices or raw devices. If using a device-mapper multipath -device, the fence_scsi agent will find the underlying devices (paths) -and created registrations for each path. - -.SH OPTIONS -.TP -\fB-o\fP \fIaction\fR -Fencing action. This value can be "on", "off", "status", or -"metadata". The "on", "off", and "status" actions require either a key -(see -k option) or node name (see -n option). For "on", the agent will -attempt to register with the device(s) and create a reservation if -none exists. The "off" action will attempt to remove a node's key from -the device(s). The "status" action will report whether or not a node's -key is currently register with one or more of the devices. The -"metadata" action will display the XML metadata. The default action if -"off". -.TP -\fB-d\fP \fIdevices\fR -List of devices to use for current operation. Devices can be -comma-separated list of raw device (eg. /dev/sdc) or device-mapper -multipath devices (eg. /dev/dm-3). Each device must support SCSI-3 -persistent reservations. -.TP -\fB-f\fP \fIlogfile\fR -Log output to file. -.TP -\fB-n\fP \fInodename\fR -Name of the node to be fenced. The node name is used to generate the -key value used for the current operation. This option will be ignored -when used with the -k option. -.TP -\fB-k\fP \fIkey\fR -Key to use for the current operation. This key should be unique to a -node. For the "on" action, the key specifies the key use to register -the local node. For the "off" action, this key specifies the key to be -removed from the device(s). -.TP -\fB-H\fP \fIdelay\fR -Wait X seconds before fencing is started (Default Value: 0) - -.TP -\fB-a\fP -Use the APTPL flag for registrations. This option is only used for the -"on" action. -.TP -\fB-h\fP -Print out a help message describing available options, then exit. -.TP -\fB-v\fP -Verbose output. -.TP -\fB-V\fP -Print out a version message, then exit. - -.SH STDIN PARAMETERS -.TP -\fIagent = "param"\fR -This option is used by fence_node(8) and is ignored by fence_scsi. -.TP -\fInodename = "param"\fR -Same as -n option. -.TP -\fIaction = "param" \fR -Same as -o option. -.TP -\fIdevices = "param"\fR -Same as -d option. -.TP -\fIlogfile = "param"\fR -Same as -f option -.TP -\fIkey = "param"\fR -Same as -k option. -.TP -\fIdelay = "param"\fR -Same as -H option. -.TP -\fIaptpl = "1" -Enable the APTPL flag. Default is 0 (disable). - -.SH SEE ALSO -fence(8), fence_node(8), sg_persist(8), vgs(8), cman_tool(8), cman(5) diff --git a/fence/agents/scsi/fence_scsi.pl b/fence/agents/scsi/fence_scsi.pl deleted file mode 100644 index 6808ff5..0000000 --- a/fence/agents/scsi/fence_scsi.pl +++ /dev/null @@ -1,875 +0,0 @@ -#!/usr/bin/perl - -use Cwd 'realpath'; -use File::Basename; -use File::Path; -use Getopt::Std; -use POSIX; -use B; - -#BEGIN_VERSION_GENERATION -$RELEASE_VERSION=""; -$REDHAT_COPYRIGHT=""; -$BUILD_DATE=""; -#END_VERSION_GENERATION - -my $ME = fileparse ($0, ".pl"); - -################################################################################ - -sub log_debug ($) -{ - my $time = strftime "%b %e %T", localtime; - my ($msg) = @_; - - print STDOUT "$time $ME: [debug] $msg\n" unless defined ($opt_q); - - return; -} - -sub log_error ($) -{ - my $time = strftime "%b %e %T", localtime; - my ($msg) = @_; - - print STDERR "$time $ME: [error] $msg\n" unless defined ($opt_q); - - exit (1); -} - -sub do_action_on ($@) -{ - my $self = (caller(0))[3]; - my ($node_key, @devices) = @_; - - key_write ($node_key); - - foreach $dev (@devices) { - log_error ("device $dev does not exist") if (! -e $dev); - log_error ("device $dev is not a block device") if (! -b $dev); - - if (do_register_ignore ($node_key, $dev) != 0) { - log_error ("failed to create registration (key=$node_key, device=$dev)"); - } - - if (!get_reservation_key ($dev)) { - if (do_reserve ($node_key, $dev) != 0) { - if (!get_reservation_key ($dev)) { - log_error ("failed to create reservation (key=$node_key, device=$dev)"); - } - } - } - } - - return; -} - -sub do_action_off ($@) -{ - my $self = (caller(0))[3]; - my ($node_key, @devices) = @_; - - my $host_key = key_read (); - - if ($host_key eq $node_key) { - log_error ($self); - } - - foreach $dev (@devices) { - log_error ("device $dev does not exist") if (! -e $dev); - log_error ("device $dev is not a block device") if (! -b $dev); - - my @keys = grep { /^$node_key$/i } get_registration_keys ($dev); - - if (scalar (@keys) != 0) { - do_preempt_abort ($host_key, $node_key, $dev); - } - } - - return; -} - -sub do_action_status ($@) -{ - my $self = (caller(0))[3]; - my ($node_key, @devices) = @_; - - my $dev_count = 0; - my $key_count = 0; - - foreach $dev (@devices) { - log_error ("device $dev does not exist") if (! -e $dev); - log_error ("device $dev is not a block device") if (! -b $dev); - - do_reset ($dev); - - my @keys = grep { /^$node_key$/i } get_registration_keys ($dev); - - if (scalar (@keys) != 0) { - $dev_count++; - } - } - - if ($dev_count != 0) { - exit (0); - } else { - exit (2); - } -} - -sub do_verify_on ($@) -{ - my $self = (caller(0))[3]; - my ($node_key, @devices) = @_; - my $count = 0; - - for $dev (@devices) { - my @keys = grep { /^$node_key$/i } get_registration_keys ($dev); - - ## check that our key is registered - if (scalar (@keys) == 0) { - log_debug ("failed to register key $node_key on device $dev"); - $count++; - next; - } - - ## write dev to device file once registration is verified - dev_write ($dev); - - ## check that a reservation exists - if (!get_reservation_key ($dev)) { - log_debug ("no reservation exists on device $dev"); - $count++; - } - } - - if ($count != 0) { - log_error ("$self: failed to verify $count devices"); - } -} - -sub do_verify_off ($@) -{ - my $self = (caller(0))[3]; - my ($node_key, @devices) = @_; - my $count = 0; - - for $dev (@devices) { - my @keys = grep { /^$node_key$/i } get_registration_keys ($dev); - - ## check that our key is not registered - if (scalar (@keys) != 0) { - log_debug ("failed to remove key $node_key from device $dev"); - $count++; - next; - } - - ## check that a reservation exists - if (!get_reservation_key ($dev)) { - log_debug ("no reservation exists on device $dev"); - $count++; - } - } - - if ($count != 0) { - log_error ("$self: failed to verify $count devices"); - } -} - -sub do_register ($$$) -{ - my $self = (caller(0))[3]; - my ($host_key, $node_key, $dev) = @_; - - $dev = realpath ($dev); - - if (substr ($dev, 5) =~ /^dm/) { - my @slaves = get_mpath_slaves ($dev); - foreach (@slaves) { - do_register ($node_key, $_); - } - return; - } - - log_debug ("$self (host_key=$host_key, node_key=$node_key, dev=$dev)"); - - my $cmd; - my $out; - my $err; - - do_reset ($dev); - - $cmd = "sg_persist -n -o -G -K $host_key -S $node_key -d $dev"; - $cmd .= " -Z" if (defined $opt_a); - $out = qx { $cmd 2> /dev/null }; - $err = ($?>>8); - - # if ($err != 0) { - # log_error ("$self (err=$err)"); - # } - - log_debug ("$self (err=$err)"); - - return ($err); -} - -sub do_register_ignore ($$) -{ - my $self = (caller(0))[3]; - my ($node_key, $dev) = @_; - - $dev = realpath ($dev); - - if (substr ($dev, 5) =~ /^dm/) { - my @slaves = get_mpath_slaves ($dev); - foreach (@slaves) { - do_register_ignore ($node_key, $_); - } - return; - } - - log_debug ("$self (node_key=$node_key, dev=$dev)"); - - my $cmd; - my $out; - my $err; - - do_reset ($dev); - - $cmd = "sg_persist -n -o -I -S $node_key -d $dev"; - $cmd .= " -Z" if (defined $opt_a); - $out = qx { $cmd 2> /dev/null }; - $err = ($?>>8); - - # if ($err != 0) { - # log_error ("$self (err=$err)"); - # } - - log_debug ("$self (err=$err)"); - - return ($err); -} - -sub do_reserve ($$) -{ - my $self = (caller(0))[3]; - my ($host_key, $dev) = @_; - - log_debug ("$self (host_key=$host_key, dev=$dev)"); - - my $cmd = "sg_persist -n -o -R -T 5 -K $host_key -d $dev"; - my $out = qx { $cmd 2> /dev/null }; - my $err = ($?>>8); - - # if ($err != 0) { - # log_error ("$self (err=$err)"); - # } - - log_debug ("$self (err=$err)"); - - return ($err); -} - -sub do_release ($$) -{ - my $self = (caller(0))[3]; - my ($host_key, $dev) = @_; - - log_debug ("$self (host_key=$host_key, dev=$dev)"); - - my $cmd = "sg_persist -n -o -L -T 5 -K $host_key -d $dev"; - my $out = qx { $cmd 2> /dev/null }; - my $err = ($?>>8); - - # if ($err != 0) { - # log_error ("$self (err=$err)"); - # } - - log_debug ("$self (err=$err)"); - - return ($err); -} - -sub do_preempt ($$$) -{ - my $self = (caller(0))[3]; - my ($host_key, $node_key, $dev) = @_; - - log_debug ("$self (host_key=$host_key, node_key=$node_key, dev=$dev)"); - - my $cmd = "sg_persist -n -o -P -T 5 -K $host_key -S $node_key -d $dev"; - my $out = qx { $cmd 2> /dev/null }; - my $err = ($?>>8); - - # if ($err != 0) { - # log_error ("$self (err=$err)"); - # } - - log_debug ("$self (err=$err)"); - - return ($err); -} - -sub do_preempt_abort ($$$) -{ - my $self = (caller(0))[3]; - my ($host_key, $node_key, $dev) = @_; - - log_debug ("$self (host_key=$host_key, node_key=$node_key, dev=$dev)"); - - my $cmd = "sg_persist -n -o -A -T 5 -K $host_key -S $node_key -d $dev"; - my $out = qx { $cmd 2> /dev/null }; - my $err = ($?>>8); - - # if ($err != 0) { - # log_error ("$self (err=$err)"); - # } - - log_debug ("$self (err=$err)"); - - return ($err); -} - -sub do_reset (S) -{ - my $self = (caller(0))[3]; - my ($dev) = @_; - - my $cmd = "sg_turs $dev"; - my @out = qx { $cmd 2> /dev/null }; - my $err = ($?>>8); - - ## note that it is not necessarily an error is $err is non-zero, - ## so just log the device and status and continue. - - log_debug ("$self (dev=$dev, status=$err)"); - - return ($err); -} - -sub dev_unlink () -{ - my $self = (caller(0))[3]; - my $file = "/var/run/cluster/fence_scsi.dev"; - - if (-e $file) { - unlink ($file) or die "$!\n"; - } - - return; -} - -sub dev_write ($) -{ - my $self = (caller(0))[3]; - my $file = "/var/run/cluster/fence_scsi.dev"; - my $dev = shift; - - if (! -d "/var/run/cluster") { - mkpath ("/var/run/cluster"); - } - - open (\*FILE, "+>>$file") or die "$!\n"; - - ## since the file is opened for read, write and append, - ## we need to seek to the beginning of the file before grep. - - seek (FILE, 0, 0); - - if (! grep { /^$dev$/ } <FILE>) { - print FILE "$dev\n"; - } - - close (FILE); - - return; -} - -sub key_read () -{ - my $self = (caller(0))[3]; - my $file = "/var/run/cluster/fence_scsi.key"; - my $key; - - open (\*FILE, "<$file") or die "$!\n"; - chomp ($key = <FILE>); - close (FILE); - - return ($key); -} - -sub key_write ($) -{ - my $self = (caller(0))[3]; - my $file = "/var/run/cluster/fence_scsi.key"; - my $key = shift; - - if (! -d "/var/run/cluster") { - mkpath ("/var/run/cluster"); - } - - open (\*FILE, ">$file") or die "$!\n"; - print FILE "$key\n"; - close (FILE); - - return; -} - -sub get_key ($) -{ - my $self = (caller(0))[3]; - - my $key = sprintf ("%.4x%.4x", get_cluster_id (), get_node_id ($_[0])); - - return ($key); -} - -sub get_node_id ($) -{ - my $self = (caller(0))[3]; - my $node = $_[0]; - - my $cmd = "/usr/sbin/corosync-cmapctl nodelist."; - my @out = qx { $cmd 2> /dev/null }; - my $err = ($?>>8); - - if ($err != 0) { - log_error ("$self (err=$err)"); - } - - # die "[error]: $self\n" if ($?>>8); - - foreach my $line (@out) { - chomp($line); - if ($line =~ /.(\d+?).ring._addr \(str\) = ${node}$/) { - return $1; - } - } - - log_error("$self (unable to parse output of corosync-cmapctl or node does not exist)"); -} - -sub get_cluster_id () -{ - my $self = (caller(0))[3]; - my $cluster_id; - - my $cmd = "/usr/sbin/corosync-cmapctl totem.cluster_name"; - my $out = qx { $cmd 2> /dev/null }; - my $err = ($?>>8); - - if ($err != 0) { - log_error ("$self (err=$err)"); - } - - # die "[error]: $self\n" if ($?>>8); - - chomp($out); - - if ($out =~ /=\s(.*?)$/) { - my $cluster_name = $1; - # tranform string to a number - $cluster_id = (hex B::hash($cluster_name)) % 65536; - } else { - log_error("$self (unable to parse output of corosync-cmapctl)"); - } - - return ($cluster_id); -} - -sub get_devices_clvm () -{ - my $self = (caller(0))[3]; - my @devices; - - my $cmd = "vgs --noheadings " . - " --separator : " . - " --sort pv_uuid " . - " --options vg_attr,pv_name " . - " --config 'global { locking_type = 0 } " . - " devices { preferred_names = [ \"^/dev/dm\" ] }'"; - - my @out = qx { $cmd 2> /dev/null }; - my $err = ($?>>8); - - if ($err != 0) { - log_error ("$self (err=$err)"); - } - - # die "[error]: $self\n" if ($?>>8); - - foreach (@out) { - chomp; - my ($vg_attr, $pv_name) = split (/:/, $_); - if ($vg_attr =~ /c$/) { - push (@devices, $pv_name); - } - } - - return (@devices); -} - -sub get_devices_scsi () -{ - my $self = (caller(0))[3]; - my @devices; - - opendir (\*DIR, "/sys/block/") or die "$!\n"; - @devices = grep { /^sd/ } readdir (DIR); - closedir (DIR); - - return (@devices); -} - -sub get_mpath_name ($) -{ - my $self = (caller(0))[3]; - my ($dev) = @_; - my $name; - - if ($dev =~ /^\/dev\//) { - $dev = substr ($dev, 5); - } - - open (\*FILE, "/sys/block/$dev/dm/name") or die "$!\n"; - chomp ($name = <FILE>); - close (FILE); - - return ($name); -} - -sub get_mpath_uuid ($) -{ - my $self = (caller(0))[3]; - my ($dev) = @_; - my $uuid; - - if ($dev =~ /^\/dev\//) { - $dev = substr ($dev, 5); - } - - open (\*FILE, "/sys/block/$dev/dm/uuid") or die "$!\n"; - chomp ($uuid = <FILE>); - close (FILE); - - return ($name); -} - -sub get_mpath_slaves ($) -{ - my $self = (caller(0))[3]; - my ($dev) = @_; - my @slaves; - - if ($dev =~ /^\/dev\//) { - $dev = substr ($dev, 5); - } - - opendir (\*DIR, "/sys/block/$dev/slaves/") or die "$!\n"; - - @slaves = grep { !/^\./ } readdir (DIR); - if ($slaves[0] =~ /^dm/) { - @slaves = get_mpath_slaves ($slaves[0]); - } else { - @slaves = map { "/dev/$_" } @slaves; - } - - closedir (DIR); - - return (@slaves); -} - -sub get_registration_keys ($) -{ - my $self = (caller(0))[3]; - my ($dev) = @_; - my @keys; - - my $cmd = "sg_persist -n -i -k -d $dev"; - my @out = qx { $cmd 2> /dev/null }; - my $err = ($?>>8); - - if ($err != 0) { - log_error ("$self (err=$err)"); - } - - # die "[error]: $self\n" if ($?>>8); - - foreach (@out) { - chomp; - if ($_ =~ s/^\s+0x//i) { - push (@keys, $_); - } - } - - return (@keys); -} - -sub get_reservation_key ($) -{ - my $self = (caller(0))[3]; - my ($dev) = @_; - my $key; - - my $cmd = "sg_persist -n -i -r -d $dev"; - my @out = qx { $cmd 2> /dev/null }; - my $err = ($?>>8); - - if ($err != 0) { - log_error ("$self (err=$err)"); - } - - # die "[error]: $self\n" if ($?>>8); - - foreach (@out) { - chomp; - if ($_ =~ s/^\s+key=0x//i) { - $key = $_; - last; - } - } - - return ($key) -} - -sub get_options_stdin () -{ - my $num = 0; - - while (<STDIN>) { - chomp; - s/^\s*//; - s/\s*$//; - - next if (/^#/); - - $num++; - - next unless ($_); - - my ($opt, $arg) = split (/\s*=\s*/, $_); - - if ($opt eq "") { - exit (1); - } - elsif ($opt eq "aptpl") { - $opt_a = $arg; - } - elsif ($opt eq "devices") { - $opt_d = $arg; - } - elsif ($opt eq "logfile") { - $opt_f = $arg; - } - elsif ($opt eq "key") { - $opt_k = $arg; - } - elsif ($opt eq "nodename") { - $opt_n = $arg; - } - elsif ($opt eq "action") { - $opt_o = $arg; - } - elsif ($opt eq "delay") { - $opt_H = $arg; - } - } -} - -sub print_usage () -{ - print "Usage:\n"; - print "\n"; - print "$ME [options]\n"; - print "\n"; - print "Options:\n"; - print " -a Use APTPL flag\n"; - print " -d <devices> Devices to be used for action\n"; - print " -f <logfile> File to write debug/error output\n"; - print " -H <timeout> Wait X seconds before fencing is started\n"; - print " -h Usage\n"; - print " -k <key> Key to be used for current action\n"; - print " -n <nodename> Name of node to operate on\n"; - print " -o <action> Action: off (default), on, or status\n"; - print " -q Quiet mode\n"; - print " -V Version\n"; - - exit (0); -} - -sub print_version () -{ - print "$ME $RELEASE_VERSION $BUILD_DATE\n"; - print "$REDHAT_COPYRIGHT\n" if ( $REDHAT_COPYRIGHT ); - - exit (0); -} - -sub print_metadata () -{ - print "<?xml version=\"1.0\" ?>\n"; - print "<resource-agent name=\"fence_scsi\"" . - " shortdesc=\"fence agent for SCSI-3 persistent reservations\">\n"; - print "<longdesc>fence_scsi</longdesc>\n"; - print "<vendor-url>http://www.t10.org</vendor-url>\n"; - print "<parameters>\n"; - print "\t<parameter name=\"aptpl\" unique=\"0\" required=\"0\">\n"; - print "\t\t<getopt mixed=\"-a\"/>\n"; - print "\t\t<content type=\"boolean\"/>\n"; - print "\t\t<shortdesc lang=\"en\">" . - "Use APTPL flag for registrations" . - "</shortdesc>\n"; - print "\t</parameter>\n"; - print "\t<parameter name=\"devices\" unique=\"0\" required=\"0\">\n"; - print "\t\t<getopt mixed=\"-d\"/>\n"; - print "\t\t<content type=\"string\"/>\n"; - print "\t\t<shortdesc lang=\"en\">" . - "List of devices to be used for fencing action" . - "</shortdesc>\n"; - print "\t</parameter>\n"; - print "\t<parameter name=\"logfile\" unique=\"0\" required=\"0\">\n"; - print "\t\t<getopt mixed=\"-f\"/>\n"; - print "\t\t<content type=\"string\"/>\n"; - print "\t\t<shortdesc lang=\"en\">" . - "File to write error/debug messages" . - "</shortdesc>\n"; - print "\t</parameter>\n"; - print "\t<parameter name=\"delay\" unique=\"0\" required=\"0\">\n"; - print "\t\t<getopt mixed=\"-H\"/>\n"; - print "\t\t<content type=\"string\"/>\n"; - print "\t\t<shortdesc lang=\"en\">" . - "Wait X seconds before fencing is started" . - "</shortdesc>\n"; - print "\t</parameter>\n"; - print "\t<parameter name=\"key\" unique=\"0\" required=\"0\">\n"; - print "\t\t<getopt mixed=\"-k\"/>\n"; - print "\t\t<content type=\"string\"/>\n"; - print "\t\t<shortdesc lang=\"en\">" . - "Key value to be used for fencing action" . - "</shortdesc>\n"; - print "\t</parameter>\n"; - print "\t<parameter name=\"action\" unique=\"0\" required=\"0\">\n"; - print "\t\t<getopt mixed=\"-o\"/>\n"; - print "\t\t<content type=\"string\" default=\"off\"/>\n"; - print "\t\t<shortdesc lang=\"en\">" . - "Fencing action" . - "</shortdesc>\n"; - print "\t</parameter>\n"; - print "\t<parameter name=\"nodename\" unique=\"0\" required=\"0\">\n"; - print "\t\t<getopt mixed=\"-n\"/>\n"; - print "\t\t<content type=\"string\"/>\n"; - print "\t\t<shortdesc lang=\"en\">" . - "Name of node" . - "</shortdesc>\n"; - print "\t</parameter>\n"; - print "</parameters>\n"; - print "<actions>\n"; - print "\t<action name=\"on\" on_target=\"1\" automatic=\"1\"/>\n"; - print "\t<action name=\"off\"/>\n"; - print "\t<action name=\"status\"/>\n"; - print "\t<action name=\"metadata\"/>\n"; - print "</actions>\n"; - print "</resource-agent>\n"; - - exit (0); -} - -################################################################################ - -if (@ARGV > 0) { - getopts ("ad:f:H:hk:n:o:qV") or print_usage; - print_usage if (defined $opt_h); - print_version if (defined $opt_V); -} else { - get_options_stdin (); -} - -## handle the metadata action here to avoid other parameter checks -## -if ($opt_o =~ /^metadata$/i) { - print_metadata; -} - -## if the logfile (-f) parameter was specified, open the logfile -## and redirect STDOUT and STDERR to the logfile. -## -if (defined $opt_f) { - open (LOG, ">>$opt_f") or die "$!\n"; - open (STDOUT, ">&LOG"); - open (STDERR, ">&LOG"); -} - -## verify that either key or nodename have been specified -## -if ((!defined $opt_n) && (!defined $opt_k)) { - print_usage (); -} - -## determine key value -## -if (defined $opt_k) { - $key = $opt_k; -} else { - $key = get_key ($opt_n); -} - -## verify that key is not zero -## -if (hex($key) == 0) { - log_error ("key cannot be zero"); -} - -## remove any leading zeros from key -## -if ($key =~ /^0/) { - $key =~ s/^0+//; -} - -## get devices -## -if (defined $opt_d) { - @devices = split (/\s*,\s*/, $opt_d); -} else { - @devices = get_devices_clvm (); -} - -## verify that device list is not empty -## -if (scalar (@devices) == 0) { - log_error ("no devices found"); -} - -## default action is "off" -## -if (!defined $opt_o) { - $opt_o = "off"; -} - -## Wait for defined period (-H / delay= ) -## -if ((defined $opt_H) && ($opt_H =~ /^[0-9]+/)) { - sleep($opt_H); -} - -## determine the action to perform -## -if ($opt_o =~ /^on$/i) { - do_action_on ($key, @devices); - do_verify_on ($key, @devices); -} -elsif ($opt_o =~ /^off$/i) { - do_action_off ($key, @devices); - do_verify_off ($key, @devices); -} -elsif ($opt_o =~ /^status/i) { - do_action_status ($key, @devices); -} else { - log_error ("unknown action '$opt_o'"); - exit (1); -} - -## close the logfile -## -if (defined $opt_f) { - close (LOG); -} diff --git a/fence/agents/scsi/fence_scsi.py b/fence/agents/scsi/fence_scsi.py new file mode 100644 index 0000000..f837faa --- /dev/null +++ b/fence/agents/scsi/fence_scsi.py @@ -0,0 +1,452 @@ +#!/usr/bin/python -tt + +import sys +import stat +import re +import os +import time +import logging +import atexit +import hashlib +sys.path.append("@FENCEAGENTSLIBDIR@") +from fencing import fail_usage, run_command, atexit_handler, check_input, process_input, show_docs, fence_action, all_opt + +#BEGIN_VERSION_GENERATION +RELEASE_VERSION="" +REDHAT_COPYRIGHT="" +BUILD_DATE="" +#END_VERSION_GENERATION + +STORE_PATH = "/var/run/cluster/fence_scsi" + + +def get_status(conn, options): + del conn + status = "off" + for dev in options["devices"]: + is_block_device(dev) + reset_dev(options, dev) + if options["--key"] in get_registration_keys(options, dev): + status = "on" + else: + logging.debug("No registration for key "\ + + options["--key"] + " on device " + dev + "\n") + return status + + +def set_status(conn, options): + del conn + count = 0 + if options["--action"] == "on": + set_key(options) + for dev in options["devices"]: + is_block_device(dev) + + register_dev(options, dev) + if options["--key"] not in get_registration_keys(options, dev): + count += 1 + logging.debug("Failed to register key "\ + + options["--key"] + "on device " + dev + "\n") + continue + dev_write(dev, options) + + if get_reservation_key(options, dev) is None \ + and not reserve_dev(options, dev) \ + and get_reservation_key(options, dev) is None: + count += 1 + logging.debug("Failed to create reservation (key="\ + + options["--key"] + ", device=" + dev + ")\n") + + else: + host_key = get_key() + if host_key == options["--key"].lower(): + fail_usage("Failed: keys cannot be same. You can not fence yourself.") + for dev in options["devices"]: + is_block_device(dev) + + if options["--key"] in get_registration_keys(options, dev): + preempt_abort(options, host_key, dev) + + for dev in options["devices"]: + if options["--key"] in get_registration_keys(options, dev): + count += 1 + logging.debug("Failed to remove key "\ + + options["--key"] + " on device " + dev + "\n") + continue + + if not get_reservation_key(options, dev): + count += 1 + logging.debug("No reservation exists on device " + dev + "\n") + if count: + logging.error("Failed to verify " + str(count) + " device(s)") + sys.exit(1) + + +#run command, returns dict, ret["err"] = exit code; ret["out"] = output +def run_cmd(options, cmd): + ret = {} + (ret["err"], ret["out"], _) = run_command(options, cmd) + ret["out"] = "".join([i for i in ret["out"] if i is not None]) + return ret + + +# check if device exist and is block device +def is_block_device(dev): + if not os.path.exists(dev): + fail_usage("Failed: device \"" + dev + "\" does not exist") + if not stat.S_ISBLK(os.stat(dev).st_mode): + fail_usage("Failed: device \"" + dev + "\" is not a block device") + + +# cancel registration +def preempt_abort(options, host, dev): + cmd = options["--sg_persist-path"] + " -n -o -A -T 5 -K " + host + " -S " + options["--key"] + " -d " + dev + return not bool(run_cmd(options, cmd)["err"]) + + +def reset_dev(options, dev): + return run_cmd(options, options["--sg_turs-path"] + " " + dev)["err"] + + +def register_dev(options, dev): + dev = os.path.realpath(dev) + if re.search(r"^dm", dev[5:]): + for slave in get_mpath_slaves(dev): + register_dev(options, slave) + return True + reset_dev(options, dev) + cmd = options["--sg_persist-path"] + " -n -o -I -S " + options["--key"] + " -d " + dev + cmd += " -Z" if "--aptpl" in options else "" + #cmd return code != 0 but registration can be successful + return not bool(run_cmd(options, cmd)["err"]) + + +def reserve_dev(options, dev): + cmd = options["--sg_persist-path"] + " -n -o -R -T 5 -K " + options["--key"] + " -d " + dev + return not bool(run_cmd(options, cmd)["err"]) + + +def get_reservation_key(options, dev): + cmd = options["--sg_persist-path"] + " -n -i -r -d " + dev + out = run_cmd(options, cmd) + if out["err"]: + fail_usage("Cannot get reservation key") + match = re.search(r"\s+key=0x(\S+)\s+", out["out"], re.IGNORECASE) + return match.group(1) if match else None + + +def get_registration_keys(options, dev): + keys = [] + cmd = options["--sg_persist-path"] + " -n -i -k -d " + dev + out = run_cmd(options, cmd) + if out["err"]: + fail_usage("Cannot get registration keys") + for line in out["out"].split("\n"): + match = re.search(r"\s+0x(\S+)\s*", line) + if match: + keys.append(match.group(1)) + return keys + + +def get_cluster_id(options): + cmd = options["--corosync-cmap-path"] + " totem.cluster_name" + + match = re.search(r"\(str\) = (\S+)\n", run_cmd(options, cmd)["out"]) + return hashlib.md5(match.group(1)).hexdigest() if match else fail_usage("Failed: cannot get cluster name") + + +def get_node_id(options): + cmd = options["--corosync-cmap-path"] + " nodelist." + + match = re.search(r".(\d).ring._addr \(str\) = " + options["--nodename"] + "\n", run_cmd(options, cmd)["out"]) + return match.group(1) if match else fail_usage("Failed: unable to parse output of corosync-cmapctl or node does not exist") + + +def generate_key(options): + return "%.4s%.4d" % (get_cluster_id(options), int(get_node_id(options))) + + +# save node key to file +def set_key(options): + file_path = options["store_path"] + ".key" + if not os.path.isdir(os.path.dirname(options["store_path"])): + os.makedirs(os.path.dirname(options["store_path"])) + try: + f = open(file_path, "w") + except IOError: + fail_usage("Failed: Cannot open file \""+ file_path + "\"") + f.write(options["--key"].lower() + "\n") + f.close() + + +# read node key from file +def get_key(): + file_path = STORE_PATH + ".key" + try: + f = open(file_path, "r") + except IOError: + fail_usage("Failed: Cannot open file \""+ file_path + "\"") + return f.readline().strip().lower() + + +def dev_write(dev, options): + file_path = options["store_path"] + ".dev" + if not os.path.isdir(os.path.dirname(options["store_path"])): + os.makedirs(os.path.dirname(options["store_path"])) + try: + f = open(file_path, "a+") + except IOError: + fail_usage("Failed: Cannot open file \""+ file_path + "\"") + out = f.read() + if not re.search(r"^" + dev + "\s+", out): + f.write(dev + "\n") + f.close() + + +def dev_read(): + file_path = STORE_PATH + ".dev" + try: + f = open(file_path, "r") + except IOError: + fail_usage("Failed: Cannot open file \"" + file_path + "\"") + # get not empty lines from file + devs = [line.strip() for line in f if line.strip()] + f.close() + return devs + + +def dev_delete(options): + file_path = options["store_path"] + ".dev" + os.remove(file_path) if os.path.exists(file_path) else None + + +def get_clvm_devices(options): + devs = [] + cmd = options["--vgs-path"] + " " +\ + "--noheadings " +\ + "--separator : " +\ + "--sort pv_uuid " +\ + "--options vg_attr,pv_name "+\ + "--config 'global { locking_type = 0 } devices { preferred_names = [ \"^/dev/dm\" ] }'" + out = run_cmd(options, cmd) + if out["err"]: + fail_usage("Failed: Cannot get clvm devices") + for line in out["out"].split("\n"): + if 'c' in line.split(":")[0]: + devs.append(line.split(":")[1]) + return devs + + +def get_mpath_slaves(dev): + if dev[:5] == "/dev/": + dev = dev[5:] + slaves = [i for i in os.listdir("/sys/block/" + dev + "/slaves/") if i[:1] != "."] + if slaves[0][:2] == "dm": + slaves = get_mpath_slaves(slaves[0]) + else: + slaves = ["/dev/" + x for x in slaves] + return slaves + + +def define_new_opts(): + all_opt["devices"] = { + "getopt" : "d:", + "longopt" : "devices", + "help" : "-d, --devices=[devices] List of devices to use for current operation", + "required" : "0", + "shortdesc" : "List of devices to use for current operation. Devices can \ +be comma-separated list of raw device (eg. /dev/sdc) or device-mapper multipath \ +devices (eg. /dev/dm-3). Each device must support SCSI-3 persistent reservations.", + "order": 1 + } + all_opt["nodename"] = { + "getopt" : "n:", + "longopt" : "nodename", + "help" : "-n, --nodename=[nodename] Name of the node to be fenced", + "required" : "0", + "shortdesc" : "Name of the node to be fenced. The node name is used to \ +generate the key value used for the current operation. This option will be \ +ignored when used with the -k option.", + "order": 1 + } + all_opt["key"] = { + "getopt" : "k:", + "longopt" : "key", + "help" : "-k, --key=[key] Key to use for the current operation", + "required" : "0", + "shortdesc" : "Key to use for the current operation. This key should be \ +unique to a node. For the \"on\" action, the key specifies the key use to \ +register the local node. For the \"off\" action, this key specifies the key to \ +be removed from the device(s).", + "order": 1 + } + all_opt["aptpl"] = { + "getopt" : "a", + "longopt" : "aptpl", + "help" : "-a, --aptpl Use the APTPL flag for registrations", + "required" : "0", + "shortdesc" : "Use the APTPL flag for registrations. This option is only used for the 'on' action.", + "order": 1 + } + all_opt["logfile"] = { + "getopt" : "f:", + "longopt" : "logfile", + "help" : "-a, --logfile Log output (stdout and stderr) to file", + "required" : "0", + "shortdesc" : "Log output (stdout and stderr) to file", + "order": 5 + } + all_opt["corosync-cmap_path"] = { + "getopt" : "Z:", + "longopt" : "corosync-cmap-path", + "help" : "--corosync-cmap-path=[path] Path to corosync-cmapctl binary", + "required" : "0", + "shortdesc" : "Path to corosync-cmapctl binary", + "default" : "@COROSYNC_CMAPCTL_PATH@", + "order": 200 + } + all_opt["sg_persist_path"] = { + "getopt" : "X:", + "longopt" : "sg_persist-path", + "help" : "--sg_persist-path=[path] Path to sg_persist binary", + "required" : "0", + "shortdesc" : "Path to sg_persist binary", + "default" : "@SG_PERSIST_PATH@", + "order": 200 + } + all_opt["sg_turs_path"] = { + "getopt" : "I:", + "longopt" : "sg_turs-path", + "help" : "--sg_turs-path=[path] Path to sg_turs binary", + "required" : "0", + "shortdesc" : "Path to sg_turs binary", + "default" : "@SG_TURS_PATH@", + "order": 200 + } + all_opt["vgs_path"] = { + "getopt" : "J:", + "longopt" : "vgs-path", + "help" : "--vgs-path=[path] Path to vgs binary", + "required" : "0", + "shortdesc" : "Path to vgs binary", + "default" : "@VGS_PATH@", + "order": 200 + } + + +def scsi_check_get_verbose(): + try: + f = open("/etc/sysconfig/watchdog", "r") + except IOError: + return False + match = re.search(r"^\s*verbose=yes", "".join(f.readlines()), re.MULTILINE) + f.close() + return bool(match) + + +def scsi_check(): + if len(sys.argv) >= 3 and sys.argv[1] == "repair": + return int(sys.argv[2]) + options = {} + options["--sg_turs-path"] = "@SG_TURS_PATH@" + options["--sg_persist-path"] = "@SG_PERSIST_PATH@" + options["--power-timeout"] = "5" + if scsi_check_get_verbose(): + logging.getLogger().setLevel(logging.DEBUG) + devs = dev_read() + if not devs: + logging.error("No devices found") + return 0 + key = get_key() + if not key: + logging.error("Key not found") + return 0 + for dev in devs: + if key in get_registration_keys(options, dev): + logging.debug("key " + key + " registered with device " + dev) + return 0 + else: + logging.debug("key " + key + " not registered with device " + dev) + logging.debug("key " + key + " registered with any devices") + return 2 + + +def main(): + + atexit.register(atexit_handler) + + device_opt = ["no_login", "no_password", "devices", "nodename", "key",\ + "aptpl", "fabric_fencing", "on_target", "corosync-cmap_path",\ + "sg_persist_path", "sg_turs_path", "logfile", "vgs_path"] + + define_new_opts() + + all_opt["action"]["help"] = "-o, --action=[action] Action: status, off (default) or on" + all_opt["action"]["default"] = "off" + all_opt["delay"]["getopt"] = "H:" + + #fence_scsi_check + if os.path.basename(sys.argv[0]) == "fence_scsi_check": + sys.exit(scsi_check()) + + options = check_input(device_opt, process_input(device_opt)) + + docs = {} + docs["shortdesc"] = "Fence agent for SCSI persistentl reservation" + docs["longdesc"] = "fence_scsi is an I/O fencing agent that uses SCSI-3 \ +persistent reservations to control access to shared storage devices. These \ +devices must support SCSI-3 persistent reservations (SPC-3 or greater) as \ +well as the \"preempt-and-abort\" subcommand.\nThe fence_scsi agent works by \ +having each node in the cluster register a unique key with the SCSI \ +devive(s). Once registered, a single node will become the reservation holder \ +by creating a \"write exclusive, registrants only\" reservation on the \ +device(s). The result is that only registered nodes may write to the \ +device(s). When a node failure occurs, the fence_scsi agent will remove the \ +key belonging to the failed node from the device(s). The failed node will no \ +longer be able to write to the device(s). A manual reboot is required." + docs["vendorurl"] = "" + show_docs(options, docs) + + # backward compatibility layer BEGIN + if "--logfile" in options: + try: + logfile = open(options["--logfile"], 'w') + sys.stderr = logfile + sys.stdout = logfile + except IOError: + fail_usage("Failed: Unable to create file " + options["--logfile"]) + # backward compatibility layer END + + options["store_path"] = STORE_PATH + + # Input control BEGIN + if not (("--nodename" in options and options["--nodename"])\ + or ("--key" in options and options["--key"])): + fail_usage("Failed: nodename or key is required") + + if not ("--key" in options and options["--key"]): + options["--key"] = generate_key(options) + + if options["--key"] == "0" or not options["--key"]: + fail_usage("Failed: key cannot be 0") + + options["--key"] = options["--key"].lstrip('0') + + if not ("--devices" in options and options["--devices"].split(",")): + options["devices"] = get_clvm_devices(options) + else: + options["devices"] = options["--devices"].split(",") + + if not options["devices"]: + fail_usage("Failed: No devices found") + # Input control END + + # backward compatibility layer + if "--delay" in options and options["--delay"].isdigit(): + time.sleep(int(options["--delay"])) + + result = fence_action(None, options, set_status, get_status) + sys.exit(result) + +if __name__ == "__main__": + main() diff --git a/fence/agents/scsi/fence_scsi_check.pl b/fence/agents/scsi/fence_scsi_check.pl deleted file mode 100644 index 9ecd7a5..0000000 --- a/fence/agents/scsi/fence_scsi_check.pl +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/perl - -use POSIX; - -################################################################################ - -my $dev_file = "/var/run/cluster/fence_scsi.dev"; -my $key_file = "/var/run/cluster/fence_scsi.key"; - -################################################################################ - -sub log_debug ($) -{ - my $time = strftime ("%b %e %T", localtime); - my $msg = shift; - - print STDOUT "$time [$0] debug: $msg\n" if ($verbose); - - return; -} - -sub log_error ($) -{ - my $time = strftime ("%b %e %T", localtime); - my $msg = shift; - - print STDERR "$time [$0] error: $msg\n"; - - return; -} - -sub do_reset ($) -{ - my $dev = shift; - - my $cmd = "sg_turs $dev"; - my @out = qx { $cmd 2> /dev/null }; - - return; -} - -sub get_registration_keys ($) -{ - my $dev = shift; - my @keys = (); - - do_reset ($dev); - - my $cmd = "sg_persist -n -i -k -d $dev"; - my @out = qx { $cmd 2> /dev/null }; - - if ($?>>8 != 0) { - log_error ("$cmd"); - exit (0); - } - - foreach (@out) { - chomp; - if (s/^\s+0x//i) { - push (@keys, $_); - } - } - - return (@keys); -} - -sub get_reservation_keys ($) -{ - my $dev = shift; - my @keys = (); - - do_reset ($dev); - - my $cmd = "sg_persist -n -i -r -d $dev"; - my @out = qx { $cmd 2> /dev/null }; - - if ($?>>8 != 0) { - log_error ("$cmd"); - exit (0); - } - - foreach (@out) { - chomp; - if (s/^\s+key=0x//i) { - push (@keys, $_); - } - } - - return (@keys); -} - -sub get_verbose () -{ - open (\*FILE, "</etc/sysconfig/watchdog") or return; - chomp (my @opt = <FILE>); - close (FILE); - - foreach (@opt) { - next if (/^#/); - next unless ($_); - - if (/^verbose=yes$/i) { - return (1); - } - } - - return (0); -} - -sub key_read () -{ - open (\*FILE, "<$key_file") or exit (0); - chomp (my $key = <FILE>); - close (FILE); - - return ($key); -} - -sub dev_read () -{ - open (\*FILE, "<$dev_file") or exit (0); - chomp (my @dev = <FILE>); - close (FILE); - - return (@dev); -} - -################################################################################ - -if ($ARGV[0] =~ /^repair$/i) { - exit ($ARGV[1]); -} - -if (-e "/etc/sysconfig/watchdog") { - $verbose = get_verbose (); -} - -if (! -e $dev_file) { - log_debug ("$dev_file does not exit"); - exit (0); -} elsif (-z $dev_file) { - log_debug ("$dev_file is empty"); - exit (0); -} - -if (! -e $key_file) { - log_debug ("$key_file does not exist"); - exit (0); -} elsif (-z $key_file) { - log_debug ("$key_file is empty"); - exit (0); -} - -my $key = key_read (); -my @dev = dev_read (); - -foreach (@dev) { - my @keys = grep { /^$key$/i } get_registration_keys ($_); - - if (scalar (@keys) != 0) { - log_debug ("key $key registered with device $_"); - exit (0); - } else { - log_debug ("key $key not registered with device $_"); - } -} - -log_debug ("key $key not registered with any devices"); - -exit (2); diff --git a/make/fencebuild.mk b/make/fencebuild.mk index 819ac36..1c4be6b 100644 --- a/make/fencebuild.mk +++ b/make/fencebuild.mk @@ -12,6 +12,10 @@ $(TARGET): $(SRC) -e 's#@''IPMITOOL_PATH@#${IPMITOOL_PATH}#g' \ -e 's#@''AMTTOOL_PATH@#${AMTTOOL_PATH}#g' \ -e 's#@''GNUTLSCLI_PATH@#${GNUTLSCLI_PATH}#g' \ + -e 's#@''COROSYNC_CMAPCTL_PATH@#${COROSYNC_CMAPCTL_PATH}#g' \ + -e 's#@''SG_PERSIST_PATH@#${SG_PERSIST_PATH}#g' \ + -e 's#@''SG_TURS_PATH@#${SG_TURS_PATH}#g' \ + -e 's#@''VGS_PATH@#${VGS_PATH}#g' \ > $@ if [ 0 -eq `echo "$(SRC)" | grep fence_ &> /dev/null; echo $$?` ]; then \ -- 1.8.3.1