Yuvipanda has submitted this change and it was merged.
Change subject: deployment: Combine labs/prod deployment server roles
......................................................................
deployment: Combine labs/prod deployment server roles
- Unify them into one
- Include scap::master into them as well. All deployment
servers are also scap masters so far, and they were
fairly intertwined anyway.
- Don't include mediawiki::packages alone, just include
all of mediawiki. This was happening in site.pp, might
as well add it here instead.
- Remove beta/scap modules. Enough has changed to allow
us to use the prod scap/ module itself. Also remove
the beta roles that aren't used anymore.
- Provide a beta::deployaccess class that sets up PAM
config for mwdeploy ssh access. This is terrible,
but my house is full of yak hair.
Change-Id: I3e947637b49ce2a94128e21db35798a49e8d45e8
---
M hieradata/labs/deployment-prep/common.yaml
M hieradata/labs/staging/common.yaml
M manifests/role/beta.pp
M manifests/role/deployment.pp
M manifests/site.pp
R modules/beta/manifests/deployaccess.pp
D modules/beta/manifests/scap/master.pp
D modules/beta/manifests/scap/rsync_slave.pp
M modules/beta/templates/pam-access.conf.erb
9 files changed, 43 insertions(+), 218 deletions(-)
Approvals:
Yuvipanda: Looks good to me, approved
jenkins-bot: Verified
diff --git a/hieradata/labs/deployment-prep/common.yaml
b/hieradata/labs/deployment-prep/common.yaml
index 05f9242..0c0a0ae 100644
--- a/hieradata/labs/deployment-prep/common.yaml
+++ b/hieradata/labs/deployment-prep/common.yaml
@@ -132,5 +132,6 @@
"role::url_downloader::url_downloader_ip": 10.68.16.135
"zotero::http_proxy": deployment-urldownloader.eqiad.wmflabs:8080
"role::trebuchet::deployment_server": deployment-bastion.eqiad.wmflabs
+"role::deployment::server::deployment_group": 'project-deployment-prep'
"dsh::config::group_source": 'puppet:///modules/beta/dsh/group'
"mediawiki::users::mwdeploy_pub_key": 'ssh-rsa
AAAAB3NzaC1yc2EAAAADAQABAAABAQDFwlmBBBJAr1GI+vuYjFh5vq0YIVa5fqE5DZdpzUZISlQ0Kt+9bIr2qNHIj+Jl5Bc6ZY1mkh8l693tAHVx+8tayoiFWYNs9IVsxR+iHgOOhAdDIBXaHaUattdiye5bQmdvJVXaVegckNX2gbmUCOc09jvZvlk3blKFTSEpZRU8dmpXQzKdZgaAq2VTajAegoFnuN9FbC7hzBPA+1NxFNKn94eIeFPSlo5rWr44OEb5Uy3O0B5c6WPM+IgfiygetP+yGL4cKv7qEjZ0Sxok/Rh1lBh1vP1YQ/Mc6tMV0s+kOv7Wz+P88bfU1/uWvy479OZdfh3NQqDTrLzqHwVW1vef
root@deployment-salt'
diff --git a/hieradata/labs/staging/common.yaml
b/hieradata/labs/staging/common.yaml
index 52a9ef9..51e4f1e 100644
--- a/hieradata/labs/staging/common.yaml
+++ b/hieradata/labs/staging/common.yaml
@@ -6,3 +6,6 @@
salt::master::salt_pillar_roots: { base: [ '/srv/pillars' ] }
salt::master::salt_module_roots: { base: [ '/srv/salt/_modules' ] }
salt::master::salt_returner_roots: { base: [ '/srv/salt/_returners' ] }
+role::deployment::server::deployment_group: 'project-staging'
+role::deployment::salt_masters::deployment_server: staging-tin.eqiad.wmflabs
+role::trebuchet::deployment_server: staging-tin.eqiad.wmflabs
diff --git a/manifests/role/beta.pp b/manifests/role/beta.pp
index d429b1c..ec07dbc 100644
--- a/manifests/role/beta.pp
+++ b/manifests/role/beta.pp
@@ -7,9 +7,6 @@
include beta::autoupdater
include beta::syncsiteresources
-
- # Bring scap related scripts such as mw-update-l10n
- include ::beta::scap::master
}
# To be applied on deployment-upload.eqiad.wmflabs
@@ -25,48 +22,6 @@
rule => 'proto tcp dport http ACCEPT;'
}
-}
-
-# Class: role::beta::rsync_slave
-#
-# Provision an rsync slave server for scap in beta
-#
-class role::beta::rsync_slave {
- system::role { 'role::beta::rsync_slave':
- description => 'Scap rsync fanout server'
- }
-
- require ::role::labs::lvm::srv
- include ::beta::scap::rsync_slave
-
- # FIXME: Each host that has this role applied must also be
- # manually added to the dsh group file found in
- # modules/beta/files/dsh/group/scap-proxies or scap will
- # not communicate with that host.
-}
-
-# Class: role::beta::scap_target
-#
-# Provision a target host for scap in beta
-#
-class role::beta::scap_target {
- system::role { 'role::beta::scap_target':
- description => 'Scap deployment target'
- }
-
- require ::role::labs::lvm::srv
- include ::beta::scap::target
-
- # Allow ssh inbound from deployment-bastion.eqiad.wmflabs for scap
- ferm::rule { 'deployment-bastion-scap-ssh':
- ensure => present,
- rule => "proto tcp dport ssh saddr ${::beta::config::bastion_ip}
ACCEPT;",
- }
-
- # FIXME: Each host that has this role applied must also be
- # manually added to the dsh group file found in
- # modules/beta/files/dsh/group/mediawiki-installation or scap will
- # not communicate with that host.
}
class role::beta::trebuchet_testing {
diff --git a/manifests/role/deployment.pp b/manifests/role/deployment.pp
index e535013..3e17919 100644
--- a/manifests/role/deployment.pp
+++ b/manifests/role/deployment.pp
@@ -128,15 +128,17 @@
}
}
-class role::deployment::deployment_servers::common(
+class role::deployment::server(
# Source of the key, change this if not in production, with hiera.
$key_source = 'puppet:///private/ssh/tin/mwdeploy_rsa',
+ $apache_fqdn = $::fqdn,
+ $deployment_group = 'wikidev',
) {
# Can't include this while scap is present on tin:
# include misc::deployment::scripts
class { 'deployment::deployment_server':
- deployer_groups => ['wikidev'],
+ deployer_groups => [$deployment_group],
}
# set umask for wikidev users so that newly-created files are g+w
@@ -151,8 +153,24 @@
include ::apache
include ::apache::mod::dav
include ::apache::mod::dav_fs
+ include ::apache::helper_scripts
- class { 'mediawiki::packages': }
+ include network::constants
+ $deployable_networks = $::network::constants::deployable_networks
+
+ include role::scap::master
+ include mediawiki
+
+ if $::realm != 'labs' {
+ include wikitech::wiki::passwords
+ }
+
+ ferm::service { 'http_deployment_server':
+ desc => 'http on trebuchet deployment servers, for serving actual
files to deploy',
+ proto => 'tcp',
+ port => '80',
+ srange => $deployable_networks,
+ }
#RT 7427
::monitoring::icinga::git_merge { 'mediawiki_config':
@@ -161,29 +179,17 @@
remote_branch => 'readonly/master'
}
- class { '::keyholder': trusted_group => 'wikidev', } ->
+ class { '::keyholder': trusted_group => $deployment_group, } ->
class { '::keyholder::monitoring': } ->
keyholder::private_key { 'mwdeploy_rsa':
source => $key_source,
}
-}
-
-class role::deployment::deployment_servers::production {
- include role::deployment::deployment_servers::common
- include network::constants
- include wikitech::wiki::passwords
- include apache::helper_scripts
- include dsh
- include rsync::server
file { '/srv/deployment':
ensure => directory,
owner => 'trebuchet',
- group => 'wikidev',
+ group => $deployment_group,
}
-
- $deployable_networks = $::network::constants::deployable_networks
- $apache_fqdn = $::fqdn
apache::site { 'deployment':
content => template('apache/sites/deployment.erb'),
@@ -198,7 +204,17 @@
ferm::service { 'deployment-redis':
proto => 'tcp',
- port => '6379',
+ port => '6379',
+ }
+
+ sudo::group { "${deployment_group}_deployment_server":
+ group => $deployment_group,
+ privileges => [
+ 'ALL = (root) NOPASSWD: /usr/bin/salt-call -l quiet --out=json
pillar.data',
+ 'ALL = (root) NOPASSWD: /usr/bin/salt-call -l quiet publish.runner
deploy.fetch *',
+ 'ALL = (root) NOPASSWD: /usr/bin/salt-call -l quiet publish.runner
deploy.checkout *',
+ 'ALL = (root) NOPASSWD: /usr/bin/salt-call -l quiet --out=json
publish.runner deploy.restart *',
+ ],
}
package { 'percona-toolkit':
@@ -209,16 +225,6 @@
# determining the state of git repos during deployments.
package { 'tig':
ensure => latest,
- }
-
- sudo::group { 'wikidev_deployment_server':
- group => 'wikidev',
- privileges => [
- 'ALL = (root) NOPASSWD: /usr/bin/salt-call -l quiet --out=json
pillar.data',
- 'ALL = (root) NOPASSWD: /usr/bin/salt-call -l quiet publish.runner
deploy.fetch *',
- 'ALL = (root) NOPASSWD: /usr/bin/salt-call -l quiet publish.runner
deploy.checkout *',
- 'ALL = (root) NOPASSWD: /usr/bin/salt-call -l quiet --out=json
publish.runner deploy.restart *',
- ],
}
}
@@ -244,58 +250,6 @@
class { 'deployment::salt_master':
repo_config => $role::deployment::config::repo_config,
deployment_config => $deployment_config,
- }
-}
-
-class role::deployment::deployment_servers::labs {
- include role::deployment::deployment_servers::common
-
- # Enable multiple test environments within a single project
- if ( $::deployment_server_override != undef ) {
- $apache_fqdn = $::deployment_server_override
- } else {
- $apache_fqdn = "${::instanceproject}-deploy.eqiad.wmflabs"
- }
-
- $deployable_networks = '10.0.0.0/8'
-
- file { '/srv/deployment':
- ensure => directory,
- owner => 'trebuchet',
- group => "project-${::instanceproject}",
- }
-
- apache::site { 'deployment':
- content => template('apache/sites/deployment.erb'),
- require => File['/srv/deployment'],
- }
-
- ferm::service { 'http_deployment_server':
- desc => 'http on trebuchet deployment servers, for serving actual
files to deploy',
- proto => 'tcp',
- port => '80',
- srange => $deployable_networks,
- }
-
- class { 'redis':
- dir => '/srv/redis',
- maxmemory => '500Mb',
- monitor => false,
- }
-
- ferm::service { 'deployment-redis':
- proto => 'tcp',
- port => '6379',
- }
-
- sudo::group { "project_${::instanceproject}_deployment_server":
- group => "project-${::instanceproject}",
- privileges => [
- 'ALL = (root) NOPASSWD: /usr/bin/salt-call -l quiet --out=json
pillar.data',
- 'ALL = (root) NOPASSWD: /usr/bin/salt-call -l quiet publish.runner
deploy.fetch *',
- 'ALL = (root) NOPASSWD: /usr/bin/salt-call -l quiet publish.runner
deploy.checkout *',
- 'ALL = (root) NOPASSWD: /usr/bin/salt-call -l quiet --out=json
publish.runner deploy.restart *',
- ],
}
}
diff --git a/manifests/site.pp b/manifests/site.pp
index 4f770b9..2cbbcdb 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -2387,9 +2387,7 @@
$cluster = 'misc'
include standard
- include role::deployment::deployment_servers::production
- include mediawiki
- include role::scap::master
+ include role::deployment::server
include mysql
include role::labsdb::manager
include ssh::hostkeys-collect
diff --git a/modules/beta/manifests/scap/target.pp
b/modules/beta/manifests/deployaccess.pp
similarity index 62%
rename from modules/beta/manifests/scap/target.pp
rename to modules/beta/manifests/deployaccess.pp
index 47ecd5d..51de78f 100644
--- a/modules/beta/manifests/scap/target.pp
+++ b/modules/beta/manifests/deployaccess.pp
@@ -1,17 +1,6 @@
-# == Class: beta::scap::target
-#
-# Provisions scap components for a scap target node.
-#
-class beta::scap::target {
- include ::beta::config
- include ::mediawiki::scap
- include ::mediawiki::users
-
- # Install authorized_keys for mwdeploy user
- ssh::userkey { 'mwdeploy':
- source => 'puppet:///private/scap/id_rsa.pub',
- }
-
+class beta::deployaccess(
+ $bastion_ip = '10.68.16.58', # ip of deployment-bastion
+) {
# Hack to replace /etc/security/access.conf (which is managed by the
# ldap::client class) with a modified version that includes an access
# grant for the mwdeploy user to authenticate from deployment-bastion.
@@ -27,4 +16,3 @@
require => File['/etc/security/access.conf~'],
}
}
-
diff --git a/modules/beta/manifests/scap/master.pp
b/modules/beta/manifests/scap/master.pp
deleted file mode 100644
index 0445209..0000000
--- a/modules/beta/manifests/scap/master.pp
+++ /dev/null
@@ -1,58 +0,0 @@
-# == Class: beta::scap::master
-#
-# Provisions scap components for a scap master node.
-#
-class beta::scap::master {
- include ::beta::config
- include ::beta::scap::target
- include ::scap::scripts
- include ::rsync::server
-
- # Install ssh private key for mwdeploy user
- file { '/home/mwdeploy/.ssh/id_rsa':
- owner => 'mwdeploy',
- group => 'mwdeploy',
- mode => '0600',
- source => 'puppet:///private/scap/id_rsa',
- require => File['/home/mwdeploy/.ssh'],
- }
-
- # Run an rsync server
- rsync::server::module { 'common':
- path => $::beta::config::scap_stage_dir,
- read_only => 'yes',
- hosts_allow => $::beta::config::rsync_networks,
- }
-
- ferm::service {'rsync_deployment_bastion':
- desc => 'rsyncd on deployment-bastion, the equivalent to tin in
prod',
- proto => 'tcp',
- port => '873',
- srange => $::beta::config::rsync_networks,
- }
-
- package { 'dsh':
- ensure => present
- }
-
- # Setup dsh configuration files used by scap
- file { '/etc/dsh':
- ensure => directory,
- owner => 'root',
- group => 'root',
- mode => '0444',
- source => 'puppet:///modules/beta/dsh',
- recurse => true,
- }
-
- # Install a scap runner script for commmand line or jenkins use
- # Depends on sudo-withagent from misc::deployment::scap_scripts
- file { '/usr/local/bin/wmf-beta-scap':
- owner => 'root',
- group => 'root',
- mode => '0555',
- require => File['/usr/local/bin/sudo-withagent'],
- source => 'puppet:///modules/beta/wmf-beta-scap',
- }
-
-}
diff --git a/modules/beta/manifests/scap/rsync_slave.pp
b/modules/beta/manifests/scap/rsync_slave.pp
deleted file mode 100644
index f048290..0000000
--- a/modules/beta/manifests/scap/rsync_slave.pp
+++ /dev/null
@@ -1,16 +0,0 @@
-# == Class: beta::scap::rsync_slave
-#
-# Provisions scap components for a scap slave rsync server.
-#
-class beta::scap::rsync_slave {
- include ::beta::config
- include ::beta::scap::target
- include ::rsync::server
-
- # Run an rsync server
- rsync::server::module { 'common':
- path => $::beta::config::scap_deploy_dir,
- read_only => 'yes',
- hosts_allow => $::beta::config::rsync_networks,
- }
-}
diff --git a/modules/beta/templates/pam-access.conf.erb
b/modules/beta/templates/pam-access.conf.erb
index 821e857..a46decb 100644
--- a/modules/beta/templates/pam-access.conf.erb
+++ b/modules/beta/templates/pam-access.conf.erb
@@ -3,5 +3,5 @@
# users except for members of the nova project
# that this instance is a member of:
-+ : mwdeploy : <%= scope.lookupvar('beta::config::bastion_ip') %>
++ : mwdeploy : <%= @bastion_ip %>
-:ALL EXCEPT (project-deployment-prep) root:ALL
--
To view, visit https://gerrit.wikimedia.org/r/195340
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I3e947637b49ce2a94128e21db35798a49e8d45e8
Gerrit-PatchSet: 27
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Yuvipanda <yuvipa...@gmail.com>
Gerrit-Reviewer: Thcipriani <tcipri...@wikimedia.org>
Gerrit-Reviewer: Yuvipanda <yuvipa...@gmail.com>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
