Ori.livneh has submitted this change and it was merged.
Change subject: Configure scap master and clients in beta
......................................................................
Configure scap master and clients in beta
Deploy MediaWiki code changes from deployment-bastion to the rest of the
beta cluster using scap.
beta::scap::target
* Install authorized_keys for the mwdeploy user
* Configure pam to allow the mwdeploy user to access hosts from
deployment-bastion
* Create /srv/common-local directory that scap will target
beta::scap::master
* Install ssh private key on deployment-bastion
* Setup rsync server
* Install and configure dsh
* Install wmf-beta-scap for running scap as mwdeploy easily
beta::scap::rsync_slave
* Configure rsync fanout servers for scap
beta::mwdeploy_sudo
* Manage sudo rights for mwdeploy which are needed by both
beta::scap::target and beta::autoupdate
role::beta::rsync_slave
* Mount secondary disk at /srv
* Apply beta::scap::rsync_slave
* Include beta::scap::target in role::applicationserver::common which is
used in role::applicationserver::appserver::{beta,jobrunner,videoscaler}
* Include beta::scap::master in role::beta::bastion
* Add ferm rule to beta::natfix that allows ssh from
deployment-bastion.eqiad.wmflabs
* Change misc::deployment::vars to set MW_COMMON to /srv/common-local
* Change mediawiki::packages to manage symlinks from /usr/local/apache
to various deployment directories.
This change assumes that a couple of things have been done to the
labs/private repository on the beta puppetmaster:
* files/scap/id_rsa and files/scap/id_rsa.pub are present
* File['/etc/ssh/userkeys'] in passwords::root changed to ensure file
permissions of 0755.
These non-public changes have been made as a local commit to
/var/lib/git/labs/private on deployment-salt.eqiad.wmflabs
Bug: 63746
Change-Id: I8f77e0c131daa6147c22021ef08bf7dd2eab99a2
---
M manifests/misc/deployment.pp
M manifests/role/applicationserver.pp
M manifests/role/beta.pp
A modules/beta/files/dsh/group/mediawiki-installation
A modules/beta/files/dsh/group/scap-proxies
A modules/beta/files/wmf-beta-scap
M modules/beta/manifests/autoupdater.pp
A modules/beta/manifests/config.pp
A modules/beta/manifests/mwdeploy_sudo.pp
M modules/beta/manifests/natfix.pp
A modules/beta/manifests/scap/master.pp
A modules/beta/manifests/scap/rsync_slave.pp
A modules/beta/manifests/scap/target.pp
A modules/beta/templates/pam-access.conf.erb
M modules/mediawiki/manifests/packages.pp
15 files changed, 267 insertions(+), 26 deletions(-)
Approvals:
Ori.livneh: Looks good to me, approved
jenkins-bot: Verified
diff --git a/manifests/misc/deployment.pp b/manifests/misc/deployment.pp
index 7234960..1582182 100644
--- a/manifests/misc/deployment.pp
+++ b/manifests/misc/deployment.pp
@@ -350,7 +350,10 @@
$dblist_common = "/srv/deployment/mediawiki/common/dblists"
$dblist_common_source = $dblist_common
} elsif $system == "scap" {
- $mw_common = "/usr/local/apache/common-local"
+ $mw_common = $::realm ? {
+ 'labs' => '/srv/common-local',
+ default => '/usr/local/apache/common-local',
+ }
$mw_common_source = "/a/common"
$dblist_common = $mw_common
$dblist_common_source = $mw_common_source
@@ -372,7 +375,7 @@
} else {
$mw_rsync_host = "deployment-bastion.${::site}.wmflabs"
- $mw_statsd_host = "deployment-bastion.${::site}.wmflabs"
+ $mw_statsd_host = "deployment-graphite.${::site}.wmflabs"
$mw_statsd_port = 8125
# The Apache directories must belong to the mwdeploy user known on
diff --git a/manifests/role/applicationserver.pp
b/manifests/role/applicationserver.pp
index 2b7452b..c361dcd 100644
--- a/manifests/role/applicationserver.pp
+++ b/manifests/role/applicationserver.pp
@@ -84,6 +84,14 @@
include labs_lvm
labs_lvm::volume { 'second-local-disk': mountat => '/srv' }
+ # FIXME: Each host that has this role applied must also be
+ # manually added to the dsh group file found in
+ # modules/beta/files/dsh/group/mediawiki-installation or scap will
+ # not communicate with that host.
+ class { '::beta::scap::target':
+ require => Labs_lvm::Volume['second-local-disk'],
+ }
+
if $hhvm == true {
notify { 'installing_hhvm': message => "Installing HHVM" }
include ::applicationserver::hhvm
diff --git a/manifests/role/beta.pp b/manifests/role/beta.pp
index 3307ac8..dc92b29 100644
--- a/manifests/role/beta.pp
+++ b/manifests/role/beta.pp
@@ -12,7 +12,7 @@
include beta::syncsiteresources
# Bring scap related scripts such as mw-update-l10n
- include misc::deployment::scap_scripts
+ include ::beta::scap::master
}
@@ -43,3 +43,23 @@
include ::beta::natfix
}
+
+# Class: role::beta::rsync_slave
+#
+# Provision an rsync slave server for scap in beta
+#
+class role::beta::rsync_slave {
+ include labs_lvm
+
+ labs_lvm::volume { 'second-local-disk':
+ mountat => '/srv',
+ }
+
+ # FIXME: Each host that has this role applied must also be
+ # manually added to the dsh group file found in
+ # modules/beta/files/dsh/group/mediawiki-installation or scap will
+ # not communicate with that host.
+ class { '::beta::scap::rsync_slave':
+ require => Labs_lvm::Volume['second-local-disk'],
+ }
+}
diff --git a/modules/beta/files/dsh/group/mediawiki-installation
b/modules/beta/files/dsh/group/mediawiki-installation
new file mode 100644
index 0000000..a1d5d0c
--- /dev/null
+++ b/modules/beta/files/dsh/group/mediawiki-installation
@@ -0,0 +1,6 @@
+deployment-apache01.eqiad.wmflabs
+deployment-apache02.eqiad.wmflabs
+deployment-bastion.eqiad.wmflabs
+deployment-jobrunner01.eqiad.wmflabs
+deployment-rsync01.eqiad.wmflabs
+deployment-videoscaler01.eqiad.wmflabs
diff --git a/modules/beta/files/dsh/group/scap-proxies
b/modules/beta/files/dsh/group/scap-proxies
new file mode 100644
index 0000000..4b2293a
--- /dev/null
+++ b/modules/beta/files/dsh/group/scap-proxies
@@ -0,0 +1 @@
+deployment-rsync01.eqiad.wmflabs
diff --git a/modules/beta/files/wmf-beta-scap b/modules/beta/files/wmf-beta-scap
new file mode 100755
index 0000000..a0c64bc
--- /dev/null
+++ b/modules/beta/files/wmf-beta-scap
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# This file is managed by puppet.
+#
+# Run scap on the beta cluster.
+#
+set -e
+
+# User that scap should be run as
+RUN_AS=mwdeploy
+
+# Ensure that script is run as the correct user
+[[ $(whoami) = $RUN_AS ]] || exec sudo -H -u $RUN_AS -- "$0" "$@"
+
+# Ensure that an ssh-agent is running
+if [[ -z $SSH_AUTH_SOCK ]]; then
+ echo Starting ssh-agent
+ eval $(ssh-agent)
+
+ # Add default keys
+ ssh-add
+
+ # Kill the agent when this script exits
+ trap 'trap - EXIT; [[ $SSH_AGENT_PID ]] && kill $SSH_AGENT_PID' \
+ EXIT SIGHUP SIGINT SIGQUIT SIGPIPE SIGTERM
+fi
+
+# Set a sane terminal type
+export TERM=dumb
+
+# Run scap
+/usr/local/bin/scap "$@"
diff --git a/modules/beta/manifests/autoupdater.pp
b/modules/beta/manifests/autoupdater.pp
index 49b60c8..08f0ab1 100644
--- a/modules/beta/manifests/autoupdater.pp
+++ b/modules/beta/manifests/autoupdater.pp
@@ -6,6 +6,7 @@
class beta::autoupdater {
require misc::deployment::common_scripts
+ include ::beta::mwdeploy_sudo
# Parsoid JavaScript dependencies are updated on beta via npm
package { 'npm':
@@ -18,24 +19,5 @@
mode => '0555',
require => Package['git-core'],
source => 'puppet:///modules/beta/wmf-beta-autoupdate.py',
- }
-
- # Make sure wmf-beta-autoupdate can run the l10n updater as l10nupdate
- sudo_user { 'mwdeploy' :
- privileges => [
- 'ALL = (l10nupdate) NOPASSWD:/usr/local/bin/mw-update-l10n',
- 'ALL = (l10nupdate) NOPASSWD:/usr/local/bin/mwscript',
- 'ALL = (l10nupdate) NOPASSWD:/usr/local/bin/refreshCdbJsonFiles',
- # Some script running as mwdeploy explicily use "sudo -u mwdeploy"
- # which makes Ubuntu to request a password. The following rule
- # make sure we are not going to ask the password to mwdeploy when
- # it tries to identify as mwdeploy.
- 'ALL = (mwdeploy) NOPASSWD: ALL',
-
- # mergeMessageFileList.php is run by mw-update-l10n as the apache
user
- # since https://gerrit.wikimedia.org/r/#/c/44548/
- # Let it runs mwscript and others as apache user.
- 'ALL = (apache) NOPASSWD: ALL',
- ]
}
}
diff --git a/modules/beta/manifests/config.pp b/modules/beta/manifests/config.pp
new file mode 100644
index 0000000..30d8976
--- /dev/null
+++ b/modules/beta/manifests/config.pp
@@ -0,0 +1,13 @@
+# == Class: beta::config
+#
+# Shared configuration for beta classes
+#
+class beta::config {
+ # IP address of deployment-bastion host
+ $bastion_ip = '10.68.16.58'
+
+ # Networks to allow for rsync
+ $rsync_networks = [
+ '10.68.16.0/21', # labs-eqiad
+ ]
+}
diff --git a/modules/beta/manifests/mwdeploy_sudo.pp
b/modules/beta/manifests/mwdeploy_sudo.pp
new file mode 100644
index 0000000..26ba57e
--- /dev/null
+++ b/modules/beta/manifests/mwdeploy_sudo.pp
@@ -0,0 +1,14 @@
+# == Class: beta::mwdeploy_sudo
+#
+# Manage sudo rights for the mwdeploy user.
+#
+class beta::mwdeploy_sudo {
+ # Grant mwdeploy sudo rights to run anything as itself, apache or
+ # l10nupdate. This is a subset of the rights granted to the wmdeploy group
+ # by the mediawiki::users::sudo class
+ sudo_user { 'mwdeploy' :
+ privileges => [
+ 'ALL = (apache,mwdeploy,l10nupdate) NOPASSWD: ALL',
+ ]
+ }
+}
diff --git a/modules/beta/manifests/natfix.pp b/modules/beta/manifests/natfix.pp
index 9dd749b..a0b2fb2 100644
--- a/modules/beta/manifests/natfix.pp
+++ b/modules/beta/manifests/natfix.pp
@@ -10,6 +10,7 @@
# RT #4824 - https://rt.wikimedia.org/Ticket/Display.html?id=4824
# bug #45868 - https://bugzilla.wikimedia.org/show_bug.cgi?id=45868
class beta::natfix {
+ include beta::config
# List out the instance public IP and private IP as described in OpenStack
# manager interface
@@ -33,4 +34,11 @@
}
create_resources( 'beta::natdestrewrite', $nat_mappings )
+
+ # Allow ssh inbound from deployment-bastion.eqiad.wmflabs for scap
+ ferm::rule { 'deployment-bastion-scap-ssh':
+ ensure => present,
+ rule => "proto tcp dport ssh saddr ${::beta::config::bastion_ip}
ACCEPT;",
+ require => Ferm::Rule['bastion-ssh'],
+ }
}
diff --git a/modules/beta/manifests/scap/master.pp
b/modules/beta/manifests/scap/master.pp
new file mode 100644
index 0000000..b374526
--- /dev/null
+++ b/modules/beta/manifests/scap/master.pp
@@ -0,0 +1,53 @@
+# == Class: beta::scap::master
+#
+# Provisions scap components for a scap master node.
+#
+class beta::scap::master {
+ include ::beta::config
+ include ::beta::scap::target
+ include rsync::server
+
+ # Install ssh private key for mwdeploy user
+ file { '/var/lib/mwdeploy/.ssh':
+ ensure => directory,
+ owner => 'mwdeploy',
+ group => 'mwdeploy',
+ mode => '0700',
+ }
+ file { '/var/lib/mwdeploy/.ssh/id_rsa':
+ owner => 'mwdeploy',
+ group => 'mwdeploy',
+ mode => '0600',
+ source => 'puppet:///private/scap/id_rsa',
+ require => File['/var/lib/mwdeploy/.ssh'],
+ }
+
+ # Run an rsync server
+ rsync::server::module { 'common':
+ path => '/data/project/apache/common-local',
+ read_only => 'yes',
+ hosts_allow => $::beta::config::rsync_networks,
+ }
+
+ package { 'dsh':
+ ensure => present
+ }
+
+ # Setup dsh configuration files used by scap
+ file { '/etc/dsh':
+ ensure => directory,
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ source => 'puppet:///modules/beta/dsh',
+ recurse => true,
+ }
+
+ # Install a scap runner script for commmand line or jenkins use
+ file { '/usr/local/bin/wmf-beta-scap':
+ owner => 'root',
+ group => 'root',
+ mode => '0555',
+ source => 'puppet:///modules/beta/wmf-beta-scap',
+ }
+}
diff --git a/modules/beta/manifests/scap/rsync_slave.pp
b/modules/beta/manifests/scap/rsync_slave.pp
new file mode 100644
index 0000000..d318838
--- /dev/null
+++ b/modules/beta/manifests/scap/rsync_slave.pp
@@ -0,0 +1,16 @@
+# == Class: beta::scap::rsync_slave
+#
+# Provisions scap components for a scap slave rsync server.
+#
+class beta::scap::rsync_slave {
+ include ::beta::config
+ include ::beta::scap::target
+ include rsync::server
+
+ # Run an rsync server
+ rsync::server::module { 'common':
+ path => '/srv/common-local',
+ read_only => 'yes',
+ hosts_allow => $::beta::config::rsync_networks,
+ }
+}
diff --git a/modules/beta/manifests/scap/target.pp
b/modules/beta/manifests/scap/target.pp
new file mode 100644
index 0000000..b22d6c8
--- /dev/null
+++ b/modules/beta/manifests/scap/target.pp
@@ -0,0 +1,56 @@
+# == Class: beta::scap::target
+#
+# Provisions scap components for a scap target node.
+#
+class beta::scap::target {
+ include ::beta::config
+ include ::misc::deployment::scap_scripts
+ include ::beta::mwdeploy_sudo
+
+ # Install authorized_keys for mwdeploy user
+ file { '/etc/ssh/userkeys/mwdeploy':
+ ensure => directory,
+ owner => 'mwdeploy',
+ group => 'mwdeploy',
+ mode => '0700',
+ require => File['/etc/ssh/userkeys'],
+ }
+ file { '/etc/ssh/userkeys/mwdeploy/.ssh':
+ ensure => directory,
+ owner => 'mwdeploy',
+ group => 'mwdeploy',
+ mode => '0700',
+ require => File['/etc/ssh/userkeys/mwdeploy'],
+ }
+ file { '/etc/ssh/userkeys/mwdeploy/.ssh/authorized_keys':
+ owner => 'mwdeploy',
+ group => 'mwdeploy',
+ mode => '0600',
+ source => 'puppet:///private/scap/id_rsa.pub',
+ require => File['/etc/ssh/userkeys/mwdeploy/.ssh'],
+ }
+
+ # Hack to replace /etc/security/access.conf (which is managed by the
+ # ldap::client class) with a modified version that includes an access
+ # grant for the mwdeploy user to authenticate from deployment-bastion.
+ file { '/etc/security/access.conf~':
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ content => template('beta/pam-access.conf.erb'),
+ }
+ File <| title == '/etc/security/access.conf' |> {
+ content => undef,
+ source => '/etc/security/access.conf~',
+ require => File['/etc/security/access.conf~'],
+ }
+
+ # Target directory for scap
+ file { '/srv/common-local':
+ ensure => directory,
+ owner => 'mwdeploy',
+ group => 'mwdeploy',
+ mode => '0755',
+ }
+}
+
diff --git a/modules/beta/templates/pam-access.conf.erb
b/modules/beta/templates/pam-access.conf.erb
new file mode 100644
index 0000000..821e857
--- /dev/null
+++ b/modules/beta/templates/pam-access.conf.erb
@@ -0,0 +1,7 @@
+# This file is managed by puppet! (beta/templates/pam-access.conf.erb)
+# Disallow access to all forms of login to all
+# users except for members of the nova project
+# that this instance is a member of:
+
++ : mwdeploy : <%= scope.lookupvar('beta::config::bastion_ip') %>
+-:ALL EXCEPT (project-deployment-prep) root:ALL
diff --git a/modules/mediawiki/manifests/packages.pp
b/modules/mediawiki/manifests/packages.pp
index 9c570a9..715a0ba 100644
--- a/modules/mediawiki/manifests/packages.pp
+++ b/modules/mediawiki/manifests/packages.pp
@@ -3,11 +3,34 @@
if $::realm == 'labs' {
file { '/usr/local/apache':
- ensure => link,
- target => '/data/project/apache',
- # Create link before wikimedia-task-appserver attempts
+ ensure => directory,
+ owner => 'root',
+ group => 'root',
+ mode => '0755',
+ }
+ file { '/usr/local/apache/common-local':
+ ensure => link,
+ # Link to files managed by scap
+ target => '/srv/common-local',
+ # Create before wikimedia-task-appserver attempts
# to create /usr/local/apache/common.
- before => Package['wikimedia-task-appserver'],
+ before => Package['wikimedia-task-appserver'],
+ require => File['/usr/local/apache'],
+ }
+ file { '/usr/local/apache/common':
+ ensure => link,
+ target => '/usr/local/apache/common-local',
+ require => File['/usr/local/apache/common-local'],
+ }
+ file { '/usr/local/apache/conf':
+ ensure => link,
+ target => '/data/project/apache/conf',
+ require => File['/usr/local/apache'],
+ }
+ file { '/usr/local/apache/uncommon':
+ ensure => link,
+ target => '/data/project/apache/uncommon',
+ require => File['/usr/local/apache'],
}
}
--
To view, visit https://gerrit.wikimedia.org/r/123674
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I8f77e0c131daa6147c22021ef08bf7dd2eab99a2
Gerrit-PatchSet: 20
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BryanDavis <[email protected]>
Gerrit-Reviewer: BryanDavis <[email protected]>
Gerrit-Reviewer: Dzahn <[email protected]>
Gerrit-Reviewer: Hashar <[email protected]>
Gerrit-Reviewer: Ori.livneh <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
