Giuseppe Lavagetto has submitted this change and it was merged.
Change subject: Conftool: Create script that checks the state after (de)pooling
......................................................................
Conftool: Create script that checks the state after (de)pooling
On certain occasions, hosts do not get (de)pooled from Pybal. This patch
adds a script to all hosts that include conftool::scripts that checks
the LVS state after issuing a command. The script is configurable in
that it can wait up to a certain interval before retrying the command,
and can also repeat the whole procedure up to N times (if needed).
This commit also adds service::node wrapper scripts for (de)pooling a
specific service.
Bug: T145518
Change-Id: If1be5238493960d272ba692094cc900c6aa5e779
---
M manifests/role/apertium.pp
M manifests/role/citoid.pp
M manifests/role/cxserver.pp
M manifests/role/graphoid.pp
M manifests/role/mathoid.pp
M manifests/role/mobileapps.pp
A modules/conftool/files/pooler_loop.rb
M modules/conftool/manifests/scripts.pp
A modules/conftool/manifests/scripts/service.pp
A modules/conftool/templates/depool-service.erb
A modules/conftool/templates/pool-service.erb
M modules/role/manifests/ores/web.pp
M modules/role/manifests/restbase/server.pp
13 files changed, 235 insertions(+), 1 deletion(-)
Approvals:
Giuseppe Lavagetto: Looks good to me, approved
jenkins-bot: Verified
diff --git a/manifests/role/apertium.pp b/manifests/role/apertium.pp
index ba208e1..5ab7043 100644
--- a/manifests/role/apertium.pp
+++ b/manifests/role/apertium.pp
@@ -6,6 +6,13 @@
description => 'Apertium APY server'
}
+ # LVS pooling/depoling scripts
+ include ::lvs::configuration
+ conftool::scripts::service { 'apertium':
+ lvs_services_config => $::lvs::configuration::lvs_services,
+ lvs_class_hosts => $::lvs::configuration::lvs_class_hosts,
+ }
+
include ::apertium
ferm::service { 'apertium_http':
diff --git a/manifests/role/citoid.pp b/manifests/role/citoid.pp
index 5fa0c69f..f30066d 100644
--- a/manifests/role/citoid.pp
+++ b/manifests/role/citoid.pp
@@ -3,5 +3,12 @@
system::role { 'role::citoid': }
+ # LVS pooling/depoling scripts
+ include ::lvs::configuration
+ conftool::scripts::service { 'citoid':
+ lvs_services_config => $::lvs::configuration::lvs_services,
+ lvs_class_hosts => $::lvs::configuration::lvs_class_hosts,
+ }
+
include ::citoid
}
diff --git a/manifests/role/cxserver.pp b/manifests/role/cxserver.pp
index ef77efb..124749e 100644
--- a/manifests/role/cxserver.pp
+++ b/manifests/role/cxserver.pp
@@ -4,6 +4,12 @@
system::role { 'role::cxserver':
description => 'content translation server'
}
+ # LVS pooling/depoling scripts
+ include ::lvs::configuration
+ conftool::scripts::service { 'cxserver':
+ lvs_services_config => $::lvs::configuration::lvs_services,
+ lvs_class_hosts => $::lvs::configuration::lvs_class_hosts,
+ }
include ::passwords::cxserver
$yandex_api_key = $::passwords::cxserver::yandex_api_key
diff --git a/manifests/role/graphoid.pp b/manifests/role/graphoid.pp
index 2c04e77..655e974 100644
--- a/manifests/role/graphoid.pp
+++ b/manifests/role/graphoid.pp
@@ -5,5 +5,12 @@
description => 'node.js service converting graph definitions into PNG'
}
+ # LVS pooling/depoling scripts
+ include ::lvs::configuration
+ conftool::scripts::service { 'graphoid':
+ lvs_services_config => $::lvs::configuration::lvs_services,
+ lvs_class_hosts => $::lvs::configuration::lvs_class_hosts,
+ }
+
include ::graphoid
}
diff --git a/manifests/role/mathoid.pp b/manifests/role/mathoid.pp
index e695b25..da8de0e 100644
--- a/manifests/role/mathoid.pp
+++ b/manifests/role/mathoid.pp
@@ -5,5 +5,12 @@
description => 'mathoid server'
}
+ # LVS pooling/depoling scripts
+ include ::lvs::configuration
+ conftool::scripts::service { 'mathoid':
+ lvs_services_config => $::lvs::configuration::lvs_services,
+ lvs_class_hosts => $::lvs::configuration::lvs_class_hosts,
+ }
+
include ::mathoid
}
diff --git a/manifests/role/mobileapps.pp b/manifests/role/mobileapps.pp
index 1887441..c05ede8 100644
--- a/manifests/role/mobileapps.pp
+++ b/manifests/role/mobileapps.pp
@@ -5,6 +5,11 @@
description => 'A service for use by mobile apps. Provides DOM
manipulation, aggregation, JSON flattening'
}
+ include ::lvs::configuration
+ conftool::scripts::service { 'mobileapps':
+ lvs_services_config => $::lvs::configuration::lvs_services,
+ lvs_class_hosts => $::lvs::configuration::lvs_class_hosts,
+ }
+
include ::mobileapps
}
-
diff --git a/modules/conftool/files/pooler_loop.rb
b/modules/conftool/files/pooler_loop.rb
new file mode 100755
index 0000000..856550a
--- /dev/null
+++ b/modules/conftool/files/pooler_loop.rb
@@ -0,0 +1,128 @@
+#!/usr/bin/env ruby
+require 'English'
+require 'optparse'
+require 'net/http'
+require 'facter'
+
+
+PATH_ROOT = '/usr/local/bin/'
+
+# default values
+args = {
+ host: Facter.value(:fqdn),
+ command: PATH_ROOT + 'pool',
+ pool: true,
+ depool: false,
+ lvs_uri: [],
+ pool_name: nil,
+ pool_subject: '',
+ interval: 1,
+ wait: 9,
+ cycles: 3
+}
+
+# parse the command-line arguments
+OptionParser.new do |opts|
+ opts.banner = "Usage: pooler-loop [options] [service]"
+ opts.on('-p', '--pool', '(Re)pool the host') do
+ args[:command] = PATH_ROOT + 'pool'
+ args[:pool] = true
+ args[:depool] = false
+ end
+ opts.on('-d', '--depool', 'Depool the host') do
+ args[:command] = PATH_ROOT + 'depool'
+ args[:depool] = true
+ args[:pool] = false
+ end
+ opts.on('-l', '--lvs-ips IP1,IP2', 'The LVS IPs to contact') do |ips|
+ args[:lvs_uris] = ips.split(/,\s*/).map{ |ip| [ip, 9090]}
+ end
+ opts.on('-P', '--pool-name POOL', 'The pool name to use') do |pool|
+ args[:pool_name] = pool
+ end
+ opts.on('-i', '--interval INTERVAL', Float,
+ 'The check interval, in seconds') do |i|
+ args[:interval] = i
+ end
+ opts.on('-w', '--wait TIMES', Integer, 'The number of intervals to wait ' +
+ 'before', 're-issuing the (de)pool command') do |w|
+ args[:wait] = w
+ end
+ opts.on('-c', '--cycles N', Integer, 'The number of times to repeat',
+ 'the process for') do |n|
+ args[:cycles] = n
+ end
+ opts.on_tail('-h', '--help', 'Show this text and exit') do
+ puts opts
+ exit
+ end
+end.parse!
+
+args[:pool_subject] = ARGV.shift || ''
+
+unless args[:pool_name]
+ puts 'You have to specify the pool name! Use pooler-loop -h for help'
+ exit 1
+end
+
+class PybalError < StandardError
+end
+
+def check_pooled_state(ip, port, pool, host, want_pooled)
+ resp = Net::HTTP.start(ip, port) do |http|
+ http.get "/pools/#{pool}/#{host}"
+ end
+ # ignore 404s
+ # rubocop:disable Style/CaseEquality
+ return true unless resp === Net::HTTPSuccess
+ # rubocop:enable Style/CaseEquality
+ enabled, active, pooled = resp.body.strip.split '/'
+ if want_pooled
+ # A pooled server must be enabled/up/pooled
+ # anything else is not ok
+ if enabled != 'enabled'
+ raise PybalError, "#{host} not logically pooled, lb #{ip}"
+ elsif active != 'up'
+ raise PybalError, "The service is not up on #{host}, lb #{ip}"
+ elsif pooled != 'pooled'
+ raise PybalError, "Service not pooled"
+ end
+ else
+ # A disabled server should be disabled/{up,down}/not pooled
+ if enabled == 'enabled'
+ raise PybalError, "#{host} is still logically pooled, lb #{ip}"
+ elsif pooled == 'pooled'
+ raise PybalError, "#{host} is still pooled, maybe too many depooled? lb
#{ip}"
+ end
+ end
+end
+
+
+# do the whole procedure args[:cycles] times
+(1..args[:cycles]).each do
+ # try to (de)pool the host we are on
+ `#{args[:command]} #{args[:pool_subject]}"`
+
+ # If the script exited with status !=0, we surely had an error; retry
directly.
+ if $CHILD_STATUS.exitstatus != 0
+ sleep args[:interval]
+ next
+ end
+ # check at most args[:wait] times each args[:interval]
+ # seconds with the LVS server to confirm the host is in
+ # the desired state
+ (1..args[:wait]).each do
+ sleep args[:interval]
+ begin
+ args[:lvs_uris].each do |uri|
+ ip, port = uri
+ check_pooled_state(ip, port, args[:pool_name], args[:host],
args[:pool])
+ end
+ exit 0
+ rescue PybalError => e
+ puts "Found an error: #{e.message}"
+ end
+ end
+end
+
+exit 2
diff --git a/modules/conftool/manifests/scripts.pp
b/modules/conftool/manifests/scripts.pp
index 0add7da..f03e3f7 100644
--- a/modules/conftool/manifests/scripts.pp
+++ b/modules/conftool/manifests/scripts.pp
@@ -15,6 +15,14 @@
class conftool::scripts {
require conftool
+ file { '/usr/local/bin/pooler-loop':
+ ensure => present,
+ owner => 'root',
+ group => 'root',
+ mode => '0555',
+ source => 'puppet:///modules/conftool/pooler_loop.rb',
+ }
+
file { '/usr/local/bin/pool':
ensure => present,
owner => 'root',
diff --git a/modules/conftool/manifests/scripts/service.pp
b/modules/conftool/manifests/scripts/service.pp
new file mode 100644
index 0000000..f411ed3
--- /dev/null
+++ b/modules/conftool/manifests/scripts/service.pp
@@ -0,0 +1,42 @@
+# Define: conftool::scripts::service
+#
+# Installs helper scripts to semi-reliably pool and depool a service host
+#
+# === Parameters
+#
+# [*lvs_name*] The name of the service in lvs terms. Defaults to $title
+#
+# [*lvs_services_config*] The service configuration for lvs.
+#
+# [*lvs_class_hosts*] The hash with lvs-class => hostnames equivalence
+#
+define conftool::scripts::service(
+ $lvs_services_config,
+ $lvs_class_hosts,
+ $lvs_name=$title,
+){
+ require ::conftool::scripts
+ $lvs_config = $lvs_services_config[$lvs_name]
+
+ $service = $lvs_config['service']
+ $port = $lvs_config['port']
+ $hostnames = $lvs_class_hosts[$lvs_config['class']]
+ $lvs_ips = inline_template(
+ "<%= @hostnames.map{|h| scope.function_ipresolve([h])}.join(',') %>")
+ file { "/usr/local/bin/pool-${title}":
+ ensure => present,
+ owner => 'root',
+ group => 'root',
+ mode => '0555',
+ content => template('conftool/pool-service.erb'),
+ }
+
+ file { "/usr/local/bin/depool-${title}":
+ ensure => present,
+ owner => 'root',
+ group => 'root',
+ mode => '0555',
+ content => template('conftool/depool-service.erb'),
+ }
+
+}
diff --git a/modules/conftool/templates/depool-service.erb
b/modules/conftool/templates/depool-service.erb
new file mode 100755
index 0000000..9e81e47
--- /dev/null
+++ b/modules/conftool/templates/depool-service.erb
@@ -0,0 +1,2 @@
+#!/bin/sh
+/usr/local/bin/pooler-loop --depool --lvs-ips <%= @lvs_ips %> --pool-name <%=
"#{@lvs_name}_#{@port}" %> service=<%= @service %>
diff --git a/modules/conftool/templates/pool-service.erb
b/modules/conftool/templates/pool-service.erb
new file mode 100755
index 0000000..9b9d7ba
--- /dev/null
+++ b/modules/conftool/templates/pool-service.erb
@@ -0,0 +1,2 @@
+#!/bin/sh
+/usr/local/bin/pooler-loop --pool --lvs-ips <%= @lvs_ips %> --pool-name <%=
"#{@lvs_name}_#{@port}" %> service=<%= @service %>
diff --git a/modules/role/manifests/ores/web.pp
b/modules/role/manifests/ores/web.pp
index c140e1f..4b9ff77 100644
--- a/modules/role/manifests/ores/web.pp
+++ b/modules/role/manifests/ores/web.pp
@@ -1,3 +1,10 @@
class role::ores::web {
+ # LVS pooling/depoling scripts
+ include ::lvs::configuration
+ conftool::scripts::service { 'ores':
+ lvs_services_config => $::lvs::configuration::lvs_services,
+ lvs_class_hosts => $::lvs::configuration::lvs_class_hosts,
+ }
+
include ::ores::web
}
diff --git a/modules/role/manifests/restbase/server.pp
b/modules/role/manifests/restbase/server.pp
index d3971ba..b4f0157 100644
--- a/modules/role/manifests/restbase/server.pp
+++ b/modules/role/manifests/restbase/server.pp
@@ -13,6 +13,12 @@
# Add conftool scripts and credentials
include ::conftool::scripts
+ # LVS pooling/depoling scripts
+ include ::lvs::configuration
+ conftool::scripts::service { 'restbase':
+ lvs_services_config => $::lvs::configuration::lvs_services,
+ lvs_class_hosts => $::lvs::configuration::lvs_class_hosts,
+ }
# RESTBase rate limiting DHT firewall rule
$rb_hosts_ferm = join(hiera('restbase::hosts'), ' ')
--
To view, visit https://gerrit.wikimedia.org/r/310454
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: If1be5238493960d272ba692094cc900c6aa5e779
Gerrit-PatchSet: 18
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Mobrovac <mobro...@wikimedia.org>
Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org>
Gerrit-Reviewer: Mobrovac <mobro...@wikimedia.org>
Gerrit-Reviewer: Zppix <megadev44s.m...@gmail.com>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
