Catrope has uploaded a new change for review.
https://gerrit.wikimedia.org/r/163068
Change subject: WIP Citoid puppetization
......................................................................
WIP Citoid puppetization
Doesn't have a deploy repo yet
Change-Id: Ib24047737e94230dd3af529f67f4cbacdeea8b4d
---
A manifests/role/citoid.pp
M manifests/role/deployment.pp
M manifests/role/mathoid.pp
M manifests/site.pp
M modules/admin/data/data.yaml
A modules/citoid/manifests/init.pp
A modules/citoid/templates/logrotate.erb
A modules/citoid/templates/upstart.erb
A modules/citoid/tests/Makefile
A modules/citoid/tests/citoid.pp
M modules/lvs/manifests/configuration.pp
M modules/lvs/manifests/monitor.pp
12 files changed, 270 insertions(+), 2 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/68/163068/1
diff --git a/manifests/role/citoid.pp b/manifests/role/citoid.pp
new file mode 100644
index 0000000..d1b9415
--- /dev/null
+++ b/manifests/role/citoid.pp
@@ -0,0 +1,114 @@
+# vim: set ts=4 et sw=4:
+
+class role::citoid::production {
+ system::role { 'role::citoid::production':
+ description => 'citoid server'
+ }
+
+ class { '::citoid':
+ base_path => '/srv/deployment/citoid/citoid',
+ node_path => '/srv/deployment/citoid/citoid/node_modules',
+ log_dir => '/var/log/citoid',
+ require => File[ '/srv/deployment/citoid/citoid' ]
+ }
+
+ file { '/srv/deployment/citoid/citoid':
+ ensure => directory,
+ owner => 'citoid',
+ group => 'citoid',
+ mode => '0755',
+ }
+
+ # FIXME: deployment::target really needs to handle this better
+ file { [ '/srv/deployment', '/srv/deployment/citoid' ]:
+ ensure => directory,
+ owner => 'root',
+ group => 'root',
+ mode => '0755',
+ }
+
+ deployment::target { 'citoid': }
+
+ group { 'citoid':
+ ensure => present,
+ name => 'citoid',
+ system => true,
+ }
+
+ user { 'citoid':
+ gid => 'citoid',
+ home => '/srv/deployment/citoid/citoid',
+ managehome => true,
+ system => true,
+ }
+
+ ferm::service { 'citoid':
+ proto => 'tcp',
+ port => '1970'
+ }
+
+ monitor_service { 'citoid':
+ description => 'citoid',
+ check_command => 'check_http_on_port!1970',
+ }
+}
+
+class role::citoid::beta {
+ system::role { 'role::citoid::beta':
+ description => 'citoid server (on beta)'
+ }
+
+ class { '::citoid':
+ base_path => '/srv/deployment/citoid/citoid',
+ node_path => '/srv/deployment/citoid/citoid/node_modules',
+ log_dir => '/var/log/citoid',
+ require => File[ '/srv/deployment/citoid/citoid' ]
+ }
+
+ file { '/srv/deployment/citoid/citoid':
+ ensure => directory,
+ owner => 'citoid',
+ group => 'citoid',
+ mode => '0755',
+ }
+
+ # FIXME: deployment::target really needs to handle this better
+ file { [ '/srv/deployment', '/srv/deployment/citoid' ]:
+ ensure => directory,
+ owner => 'root',
+ group => 'root',
+ mode => '0755',
+ }
+
+ deployment::target { 'citoid': }
+
+ group { 'citoid':
+ ensure => present,
+ name => 'citoid',
+ system => true,
+ }
+
+ user { 'citoid':
+ gid => 'citoid',
+ home => '/srv/deployment/citoid/citoid',
+ managehome => true,
+ system => true,
+ }
+
+ # Beta citoid server has some ferm DNAT rewriting rules (bug 45868) so we
+ # have to explicitly allow citoid port 1970
+ ferm::service { 'citoid':
+ proto => 'tcp',
+ port => '1970'
+ }
+
+ # Allow ssh access from the Jenkins master to the server where citoid is
+ # running
+ include contint::firewall::labs
+
+ # Instance got to be a Jenkins slave so we can update citoid whenever a
+ # change is made on mediawiki/services/citoid repository
+ include role::ci::slave::labs::common
+ # Also need the slave scripts for multi-git.sh
+ include contint::slave-scripts
+}
diff --git a/manifests/role/deployment.pp b/manifests/role/deployment.pp
index 2c3c51c..856a482 100644
--- a/manifests/role/deployment.pp
+++ b/manifests/role/deployment.pp
@@ -64,6 +64,12 @@
'service_name' => 'mathoid',
'checkout_submodules' => true,
},
+ 'citoid/citoid' => {
+ 'grain' => 'citoid',
+ 'upstream' =>
'https://gerrit.wikimedia.org/r/mediawiki/services/citoid',
+ 'service_name' => 'citoid',
+ 'checkout_submodules' => true,
+ },
'rcstream/rcstream' => {
'grain' => 'rcstream',
'upstream' =>
'https://gerrit.wikimedia.org/r/mediawiki/services/rcstream',
diff --git a/manifests/role/mathoid.pp b/manifests/role/mathoid.pp
index 27b4908..b089888 100644
--- a/manifests/role/mathoid.pp
+++ b/manifests/role/mathoid.pp
@@ -1,6 +1,6 @@
# vim: set ts=4 et sw=4:
-# TODO: when other services inhabit service cluster A, move this definition in
a
+# TODO: now that other services inhabit service cluster A, move this
definition in a
# better place
@monitor_group { 'sca_eqiad': description => 'Service Cluster A servers' }
diff --git a/manifests/site.pp b/manifests/site.pp
index b106c9a..adff05f 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -3186,11 +3186,14 @@
include standard
include role::mathoid::production
+ include role::citoid::production
class { 'admin':
groups => [
'mathoid-roots',
- 'mathoid-admin'
+ 'mathoid-admin',
+ 'citoid-roots',
+ 'citoid-admin'
]
}
}
diff --git a/modules/admin/data/data.yaml b/modules/admin/data/data.yaml
index e2f0355..2aa6434 100644
--- a/modules/admin/data/data.yaml
+++ b/modules/admin/data/data.yaml
@@ -190,6 +190,19 @@
'ALL = (root) NOPASSWD: /usr/sbin/service mathoid start',
'ALL = (root) NOPASSWD: /usr/sbin/service mathoid restart',
'ALL = (root) NOPASSWD: /usr/sbin/service mathoid reload']
+ citoid-roots:
+ gid: 729
+ description: root rights for citoid
+ members: [gwicke]
+ privs: ['ALL = (citoid) NOPASSWD: ALL']
+ citoid-admin:
+ gid: 730
+ description: group of citoid admins
+ members: [gwicke]
+ privs: ['ALL = (root) NOPASSWD: /usr/sbin/service citoid stop',
+ 'ALL = (root) NOPASSWD: /usr/sbin/service citoid start',
+ 'ALL = (root) NOPASSWD: /usr/sbin/service citoid restart',
+ 'ALL = (root) NOPASSWD: /usr/sbin/service citoid reload']
analytics-users:
gid: 7080
description: Gives generic client access to the Analytics (Hadoop) cluster.
diff --git a/modules/citoid/manifests/init.pp b/modules/citoid/manifests/init.pp
new file mode 100644
index 0000000..1f6c9cb
--- /dev/null
+++ b/modules/citoid/manifests/init.pp
@@ -0,0 +1,61 @@
+# == Class: citoid
+#
+# citoid is a node.js backend for citation lookups.
+#
+# === Parameters
+#
+# [*base_path*]
+# Path to the citoid code.
+# [*node_path*]
+# Path to the node modules citoid depends on.
+# [*log_dir*]
+# Place where citoid can put log files. Assumed to be already existing and
+# have write access to citoid user.
+# [*port*]
+# Port where to run the citoid service. Defaults to 1970.
+#
+class citoid(
+ $base_path,
+ $node_path,
+ $log_dir,
+ $port=1970
+) {
+ package { ['nodejs']:
+ ensure => present,
+ }
+
+ $log_file = "${log_dir}/main.log"
+
+ file { $log_dir:
+ ensure => directory,
+ owner => citoid,
+ group => citoid,
+ mode => '0775',
+ }
+
+ # The upstart configuration
+ file { '/etc/init/citoid.conf':
+ ensure => present,
+ owner => root,
+ group => root,
+ mode => '0444',
+ content => template('citoid/upstart.erb'),
+ }
+
+ file { '/etc/logrotate.d/citoid':
+ ensure => present,
+ owner => root,
+ group => root,
+ mode => '0444',
+ content => template('citoid/logrotate.erb'),
+ }
+
+ service { 'citoid':
+ ensure => running,
+ hasstatus => true,
+ hasrestart => true,
+ provider => 'upstart',
+ require => File[$log_dir],
+ subscribe => File['/etc/init/citoid.conf'],
+ }
+}
diff --git a/modules/citoid/templates/logrotate.erb
b/modules/citoid/templates/logrotate.erb
new file mode 100644
index 0000000..13649f9
--- /dev/null
+++ b/modules/citoid/templates/logrotate.erb
@@ -0,0 +1,13 @@
+#####################################################################
+### THIS FILE IS MANAGED BY PUPPET
+#####################################################################
+
+<%= @log_dir %>/* {
+ daily
+ copytruncate
+ missingok
+ compress
+ notifempty
+ rotate 15
+ size 256M
+}
diff --git a/modules/citoid/templates/upstart.erb
b/modules/citoid/templates/upstart.erb
new file mode 100644
index 0000000..af9b979
--- /dev/null
+++ b/modules/citoid/templates/upstart.erb
@@ -0,0 +1,25 @@
+#####################################################################
+### THIS FILE IS MANAGED BY PUPPET
+#####################################################################
+
+description "Citoid HTTP service"
+
+start on (local-filesystems and net-device-up IFACE!=lo)
+stop on runlevel [!2345]
+
+# up ulimit -n a bit
+limit nofile 10000 10000
+
+setuid "citoid"
+setgid "citoid"
+
+env NODE_PATH="<%= @node_path %>"
+env CITOID_PORT="<%= @port %>"
+
+respawn
+
+# wait 60 seconds for a graceful restart before killing the master
+kill timeout 60
+
+chdir "<%= @base_path %>"
+exec /usr/bin/nodejs citoid.js >> "<%= @log_file %>" 2>&1
diff --git a/modules/citoid/tests/Makefile b/modules/citoid/tests/Makefile
new file mode 100644
index 0000000..76cd656
--- /dev/null
+++ b/modules/citoid/tests/Makefile
@@ -0,0 +1,13 @@
+MANIFESTS=$(wildcard *.pp)
+OBJS=$(MANIFESTS:.pp=.po)
+TESTS_DIR=$(dir $(CURDIR))
+MODULE_DIR=$(TESTS_DIR:/=)
+MODULES_DIR=$(dir $(MODULE_DIR))
+
+all: test
+
+test: $(OBJS)
+
+%.po: %.pp
+ puppet parser validate $<
+ puppet apply --noop --modulepath $(MODULES_DIR) $<
diff --git a/modules/citoid/tests/citoid.pp b/modules/citoid/tests/citoid.pp
new file mode 100644
index 0000000..9e87297
--- /dev/null
+++ b/modules/citoid/tests/citoid.pp
@@ -0,0 +1,6 @@
+class { 'citoid':
+ base_path => '/tmp/citoid/',
+ node_path => '/tmp/citoid/node_modules',
+ log_dir => '/var/log/citoid',
+ port => '1970'
+}
diff --git a/modules/lvs/manifests/configuration.pp
b/modules/lvs/manifests/configuration.pp
index 277114d..ac9e6c0 100644
--- a/modules/lvs/manifests/configuration.pp
+++ b/modules/lvs/manifests/configuration.pp
@@ -874,5 +874,18 @@
'IdleConnection' => $idleconnection_monitor_options,
},
},
+ 'citoid' => {
+ 'description' => 'Citation lookup service, citoid.svc.eqiad.wmnet',
+ 'class' => 'low-traffic',
+ 'sites' => [ 'eqiad' ],
+ 'ip' => $service_ips['citoid'][$::site],
+ 'port' => 1970,
+ 'bgp' => 'yes',
+ 'depool-threshold' => '.5',
+ 'monitors' => {
+ 'ProxyFetch' => { 'url' => [ 'http://citoid.svc.eqiad.wmnet' ]
},
+ 'IdleConnection' => $idleconnection_monitor_options,
+ }
+ }
}
}
diff --git a/modules/lvs/manifests/monitor.pp b/modules/lvs/manifests/monitor.pp
index 0b7f455..7daffd2 100644
--- a/modules/lvs/manifests/monitor.pp
+++ b/modules/lvs/manifests/monitor.pp
@@ -16,6 +16,7 @@
lvs::monitor_service_http { "search.svc.eqiad.wmnet": ip_address =>
"10.2.2.30", check_command => "check_http_on_port!9200", contact_group =>
"admins" }
lvs::monitor_service_http { 'ocg.svc.eqiad.wmnet': ip_address =>
$ip['ocg']['eqiad'], check_command =>
"check_http_lvs_on_port!ocg.svc.eqiad.wmnet!8000!/?command=health" }
lvs::monitor_service_http { 'mathoid.svc.eqiad.wmnet': ip_address =>
$ip['mathoid']['eqiad'], check_command =>
"check_http_lvs_on_port!mathoid.svc.eqiad.wmnet!10042" }
+ lvs::monitor_service_http { 'citoid.svc.eqiad.wmnet': ip_address =>
$ip['citoid']['eqiad'], check_command =>
"check_http_lvs_on_port!citoid.svc.eqiad.wmnet!1970" }
lvs::monitor_service_custom { "search-pool1.svc.eqiad.wmnet": ip_address
=> "10.2.2.11", port => 8123, description => "LVS Lucene", check_command =>
"check_lucene" }
lvs::monitor_service_custom { "search-pool2.svc.eqiad.wmnet": ip_address
=> "10.2.2.12", port => 8123, description => "LVS Lucene", check_command =>
"check_lucene" }
--
To view, visit https://gerrit.wikimedia.org/r/163068
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib24047737e94230dd3af529f67f4cbacdeea8b4d
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Catrope <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
