Giuseppe Lavagetto has submitted this change and it was merged. Change subject: Conftool: Create script that checks the state after (de)pooling ......................................................................
Conftool: Create script that checks the state after (de)pooling On certain occasions, hosts do not get (de)pooled from Pybal. This patch adds a script to all hosts that include conftool::scripts that checks the LVS state after issuing a command. The script is configurable in that it can wait up to a certain interval before retrying the command, and can also repeat the whole procedure up to N times (if needed). This commit also adds service::node wrapper scripts for (de)pooling a specific service. Bug: T145518 Change-Id: If1be5238493960d272ba692094cc900c6aa5e779 --- M manifests/role/apertium.pp M manifests/role/citoid.pp M manifests/role/cxserver.pp M manifests/role/graphoid.pp M manifests/role/mathoid.pp M manifests/role/mobileapps.pp A modules/conftool/files/pooler_loop.rb M modules/conftool/manifests/scripts.pp A modules/conftool/manifests/scripts/service.pp A modules/conftool/templates/depool-service.erb A modules/conftool/templates/pool-service.erb M modules/role/manifests/ores/web.pp M modules/role/manifests/restbase/server.pp 13 files changed, 235 insertions(+), 1 deletion(-) Approvals: Giuseppe Lavagetto: Looks good to me, approved jenkins-bot: Verified diff --git a/manifests/role/apertium.pp b/manifests/role/apertium.pp index ba208e1..5ab7043 100644 --- a/manifests/role/apertium.pp +++ b/manifests/role/apertium.pp @@ -6,6 +6,13 @@ description => 'Apertium APY server' } + # LVS pooling/depoling scripts + include ::lvs::configuration + conftool::scripts::service { 'apertium': + lvs_services_config => $::lvs::configuration::lvs_services, + lvs_class_hosts => $::lvs::configuration::lvs_class_hosts, + } + include ::apertium ferm::service { 'apertium_http': diff --git a/manifests/role/citoid.pp b/manifests/role/citoid.pp index 5fa0c69f..f30066d 100644 --- a/manifests/role/citoid.pp +++ b/manifests/role/citoid.pp @@ -3,5 +3,12 @@ system::role { 'role::citoid': } + # LVS pooling/depoling scripts + include ::lvs::configuration + conftool::scripts::service { 'citoid': + lvs_services_config => $::lvs::configuration::lvs_services, + lvs_class_hosts => $::lvs::configuration::lvs_class_hosts, + } + include ::citoid } diff --git a/manifests/role/cxserver.pp b/manifests/role/cxserver.pp index ef77efb..124749e 100644 --- a/manifests/role/cxserver.pp +++ b/manifests/role/cxserver.pp @@ -4,6 +4,12 @@ system::role { 'role::cxserver': description => 'content translation server' } + # LVS pooling/depoling scripts + include ::lvs::configuration + conftool::scripts::service { 'cxserver': + lvs_services_config => $::lvs::configuration::lvs_services, + lvs_class_hosts => $::lvs::configuration::lvs_class_hosts, + } include ::passwords::cxserver $yandex_api_key = $::passwords::cxserver::yandex_api_key diff --git a/manifests/role/graphoid.pp b/manifests/role/graphoid.pp index 2c04e77..655e974 100644 --- a/manifests/role/graphoid.pp +++ b/manifests/role/graphoid.pp @@ -5,5 +5,12 @@ description => 'node.js service converting graph definitions into PNG' } + # LVS pooling/depoling scripts + include ::lvs::configuration + conftool::scripts::service { 'graphoid': + lvs_services_config => $::lvs::configuration::lvs_services, + lvs_class_hosts => $::lvs::configuration::lvs_class_hosts, + } + include ::graphoid } diff --git a/manifests/role/mathoid.pp b/manifests/role/mathoid.pp index e695b25..da8de0e 100644 --- a/manifests/role/mathoid.pp +++ b/manifests/role/mathoid.pp @@ -5,5 +5,12 @@ description => 'mathoid server' } + # LVS pooling/depoling scripts + include ::lvs::configuration + conftool::scripts::service { 'mathoid': + lvs_services_config => $::lvs::configuration::lvs_services, + lvs_class_hosts => $::lvs::configuration::lvs_class_hosts, + } + include ::mathoid } diff --git a/manifests/role/mobileapps.pp b/manifests/role/mobileapps.pp index 1887441..c05ede8 100644 --- a/manifests/role/mobileapps.pp +++ b/manifests/role/mobileapps.pp @@ -5,6 +5,11 @@ description => 'A service for use by mobile apps. Provides DOM manipulation, aggregation, JSON flattening' } + include ::lvs::configuration + conftool::scripts::service { 'mobileapps': + lvs_services_config => $::lvs::configuration::lvs_services, + lvs_class_hosts => $::lvs::configuration::lvs_class_hosts, + } + include ::mobileapps } - diff --git a/modules/conftool/files/pooler_loop.rb b/modules/conftool/files/pooler_loop.rb new file mode 100755 index 0000000..856550a --- /dev/null +++ b/modules/conftool/files/pooler_loop.rb @@ -0,0 +1,128 @@ +#!/usr/bin/env ruby +require 'English' +require 'optparse' +require 'net/http' +require 'facter' + + +PATH_ROOT = '/usr/local/bin/' + +# default values +args = { + host: Facter.value(:fqdn), + command: PATH_ROOT + 'pool', + pool: true, + depool: false, + lvs_uri: [], + pool_name: nil, + pool_subject: '', + interval: 1, + wait: 9, + cycles: 3 +} + +# parse the command-line arguments +OptionParser.new do |opts| + opts.banner = "Usage: pooler-loop [options] [service]" + opts.on('-p', '--pool', '(Re)pool the host') do + args[:command] = PATH_ROOT + 'pool' + args[:pool] = true + args[:depool] = false + end + opts.on('-d', '--depool', 'Depool the host') do + args[:command] = PATH_ROOT + 'depool' + args[:depool] = true + args[:pool] = false + end + opts.on('-l', '--lvs-ips IP1,IP2', 'The LVS IPs to contact') do |ips| + args[:lvs_uris] = ips.split(/,\s*/).map{ |ip| [ip, 9090]} + end + opts.on('-P', '--pool-name POOL', 'The pool name to use') do |pool| + args[:pool_name] = pool + end + opts.on('-i', '--interval INTERVAL', Float, + 'The check interval, in seconds') do |i| + args[:interval] = i + end + opts.on('-w', '--wait TIMES', Integer, 'The number of intervals to wait ' + + 'before', 're-issuing the (de)pool command') do |w| + args[:wait] = w + end + opts.on('-c', '--cycles N', Integer, 'The number of times to repeat', + 'the process for') do |n| + args[:cycles] = n + end + opts.on_tail('-h', '--help', 'Show this text and exit') do + puts opts + exit + end +end.parse! + +args[:pool_subject] = ARGV.shift || '' + +unless args[:pool_name] + puts 'You have to specify the pool name! Use pooler-loop -h for help' + exit 1 +end + +class PybalError < StandardError +end + +def check_pooled_state(ip, port, pool, host, want_pooled) + resp = Net::HTTP.start(ip, port) do |http| + http.get "/pools/#{pool}/#{host}" + end + # ignore 404s + # rubocop:disable Style/CaseEquality + return true unless resp === Net::HTTPSuccess + # rubocop:enable Style/CaseEquality + enabled, active, pooled = resp.body.strip.split '/' + if want_pooled + # A pooled server must be enabled/up/pooled + # anything else is not ok + if enabled != 'enabled' + raise PybalError, "#{host} not logically pooled, lb #{ip}" + elsif active != 'up' + raise PybalError, "The service is not up on #{host}, lb #{ip}" + elsif pooled != 'pooled' + raise PybalError, "Service not pooled" + end + else + # A disabled server should be disabled/{up,down}/not pooled + if enabled == 'enabled' + raise PybalError, "#{host} is still logically pooled, lb #{ip}" + elsif pooled == 'pooled' + raise PybalError, "#{host} is still pooled, maybe too many depooled? lb #{ip}" + end + end +end + + +# do the whole procedure args[:cycles] times +(1..args[:cycles]).each do + # try to (de)pool the host we are on + `#{args[:command]} #{args[:pool_subject]}"` + + # If the script exited with status !=0, we surely had an error; retry directly. + if $CHILD_STATUS.exitstatus != 0 + sleep args[:interval] + next + end + # check at most args[:wait] times each args[:interval] + # seconds with the LVS server to confirm the host is in + # the desired state + (1..args[:wait]).each do + sleep args[:interval] + begin + args[:lvs_uris].each do |uri| + ip, port = uri + check_pooled_state(ip, port, args[:pool_name], args[:host], args[:pool]) + end + exit 0 + rescue PybalError => e + puts "Found an error: #{e.message}" + end + end +end + +exit 2 diff --git a/modules/conftool/manifests/scripts.pp b/modules/conftool/manifests/scripts.pp index 0add7da..f03e3f7 100644 --- a/modules/conftool/manifests/scripts.pp +++ b/modules/conftool/manifests/scripts.pp @@ -15,6 +15,14 @@ class conftool::scripts { require conftool + file { '/usr/local/bin/pooler-loop': + ensure => present, + owner => 'root', + group => 'root', + mode => '0555', + source => 'puppet:///modules/conftool/pooler_loop.rb', + } + file { '/usr/local/bin/pool': ensure => present, owner => 'root', diff --git a/modules/conftool/manifests/scripts/service.pp b/modules/conftool/manifests/scripts/service.pp new file mode 100644 index 0000000..f411ed3 --- /dev/null +++ b/modules/conftool/manifests/scripts/service.pp @@ -0,0 +1,42 @@ +# Define: conftool::scripts::service +# +# Installs helper scripts to semi-reliably pool and depool a service host +# +# === Parameters +# +# [*lvs_name*] The name of the service in lvs terms. Defaults to $title +# +# [*lvs_services_config*] The service configuration for lvs. +# +# [*lvs_class_hosts*] The hash with lvs-class => hostnames equivalence +# +define conftool::scripts::service( + $lvs_services_config, + $lvs_class_hosts, + $lvs_name=$title, +){ + require ::conftool::scripts + $lvs_config = $lvs_services_config[$lvs_name] + + $service = $lvs_config['service'] + $port = $lvs_config['port'] + $hostnames = $lvs_class_hosts[$lvs_config['class']] + $lvs_ips = inline_template( + "<%= @hostnames.map{|h| scope.function_ipresolve([h])}.join(',') %>") + file { "/usr/local/bin/pool-${title}": + ensure => present, + owner => 'root', + group => 'root', + mode => '0555', + content => template('conftool/pool-service.erb'), + } + + file { "/usr/local/bin/depool-${title}": + ensure => present, + owner => 'root', + group => 'root', + mode => '0555', + content => template('conftool/depool-service.erb'), + } + +} diff --git a/modules/conftool/templates/depool-service.erb b/modules/conftool/templates/depool-service.erb new file mode 100755 index 0000000..9e81e47 --- /dev/null +++ b/modules/conftool/templates/depool-service.erb @@ -0,0 +1,2 @@ +#!/bin/sh +/usr/local/bin/pooler-loop --depool --lvs-ips <%= @lvs_ips %> --pool-name <%= "#{@lvs_name}_#{@port}" %> service=<%= @service %> diff --git a/modules/conftool/templates/pool-service.erb b/modules/conftool/templates/pool-service.erb new file mode 100755 index 0000000..9b9d7ba --- /dev/null +++ b/modules/conftool/templates/pool-service.erb @@ -0,0 +1,2 @@ +#!/bin/sh +/usr/local/bin/pooler-loop --pool --lvs-ips <%= @lvs_ips %> --pool-name <%= "#{@lvs_name}_#{@port}" %> service=<%= @service %> diff --git a/modules/role/manifests/ores/web.pp b/modules/role/manifests/ores/web.pp index c140e1f..4b9ff77 100644 --- a/modules/role/manifests/ores/web.pp +++ b/modules/role/manifests/ores/web.pp @@ -1,3 +1,10 @@ class role::ores::web { + # LVS pooling/depoling scripts + include ::lvs::configuration + conftool::scripts::service { 'ores': + lvs_services_config => $::lvs::configuration::lvs_services, + lvs_class_hosts => $::lvs::configuration::lvs_class_hosts, + } + include ::ores::web } diff --git a/modules/role/manifests/restbase/server.pp b/modules/role/manifests/restbase/server.pp index d3971ba..b4f0157 100644 --- a/modules/role/manifests/restbase/server.pp +++ b/modules/role/manifests/restbase/server.pp @@ -13,6 +13,12 @@ # Add conftool scripts and credentials include ::conftool::scripts + # LVS pooling/depoling scripts + include ::lvs::configuration + conftool::scripts::service { 'restbase': + lvs_services_config => $::lvs::configuration::lvs_services, + lvs_class_hosts => $::lvs::configuration::lvs_class_hosts, + } # RESTBase rate limiting DHT firewall rule $rb_hosts_ferm = join(hiera('restbase::hosts'), ' ') -- To view, visit https://gerrit.wikimedia.org/r/310454 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: If1be5238493960d272ba692094cc900c6aa5e779 Gerrit-PatchSet: 18 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Mobrovac <mobro...@wikimedia.org> Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org> Gerrit-Reviewer: Mobrovac <mobro...@wikimedia.org> Gerrit-Reviewer: Zppix <megadev44s.m...@gmail.com> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits