Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/219136
Change subject: Update and restart services during git-update
......................................................................
Update and restart services during git-update
Thus far, running vagrant git-update would update the git repos of
services living in /vagrant/srv. This patch enhances its behaviour by
also performing a dependency update and a service restart. For a service
to do that, it must declare a service::gitupdate block specifying which
updates need to be carried out and whether the service should be
restarted. A configuration file for each service is then placed in
/vagrant/srv/.conf.d and later read by run-git-update. This patch also
updates all of the service modules to use it.
Note: this patch also contains a fix for the phabricator module,
instructing it to connect to MySQL via the TCP socket, not the UNIX one
(the latter has been disabled in the new HHVM version packaged by WMF).
Bug: T101749
Change-Id: I7ec1d2eaa3a003d4c3bfb80a29fa11fb64506407
---
M puppet/hieradata/common.yaml
M puppet/modules/arcanist/manifests/init.pp
M puppet/modules/crm/manifests/init.pp
M puppet/modules/iegreview/manifests/init.pp
M puppet/modules/mediawiki/manifests/init.pp
M puppet/modules/mediawiki/manifests/jobrunner.pp
M puppet/modules/mediawiki/manifests/parsoid.pp
M puppet/modules/mediawiki/templates/run-git-update.erb
M puppet/modules/phabricator/manifests/init.pp
M puppet/modules/role/manifests/phragile.pp
M puppet/modules/scholarships/manifests/init.pp
A puppet/modules/service/manifests/gitupdate.pp
M puppet/modules/service/manifests/init.pp
M puppet/modules/service/manifests/node.pp
A puppet/modules/service/templates/gitupdate.conf.erb
M puppet/modules/smashpig/manifests/init.pp
16 files changed, 179 insertions(+), 34 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/vagrant
refs/changes/36/219136/1
diff --git a/puppet/hieradata/common.yaml b/puppet/hieradata/common.yaml
index e907a21..ffa1b96 100644
--- a/puppet/hieradata/common.yaml
+++ b/puppet/hieradata/common.yaml
@@ -235,6 +235,7 @@
sentry::admin_pass: vagrant
service::root_dir: "%{hiera('mwv::services_dir')}"
+service::conf_dir: "%{hiera('mwv::services_dir')}/.conf.d"
service::log_dir: /vagrant/logs
service::log_level: info
diff --git a/puppet/modules/arcanist/manifests/init.pp
b/puppet/modules/arcanist/manifests/init.pp
index 6f62b7c..cdc5e7e 100644
--- a/puppet/modules/arcanist/manifests/init.pp
+++ b/puppet/modules/arcanist/manifests/init.pp
@@ -25,4 +25,13 @@
env::profile_script { 'add arcanist bin to path':
content => "export PATH=\$PATH:${deploy_dir}/arcanist/bin",
}
+
+ service::gitupdate { 'libphutil':
+ dir => "${deploy_dir}/libphutil",
+ }
+
+ service::gitupdate { 'arcanist':
+ dir => "${deploy_dir}/arcanist",
+ }
+
}
diff --git a/puppet/modules/crm/manifests/init.pp
b/puppet/modules/crm/manifests/init.pp
index 9ea1e02..c6706cd 100644
--- a/puppet/modules/crm/manifests/init.pp
+++ b/puppet/modules/crm/manifests/init.pp
@@ -46,6 +46,12 @@
require => Git::Clone[$repo],
}
+ service::gitupdate { 'crm':
+ dir => $dir,
+ type => 'php',
+ update => true,
+ }
+
# required by module ganglia_reporter
package { 'ganglia-monitor': }
diff --git a/puppet/modules/iegreview/manifests/init.pp
b/puppet/modules/iegreview/manifests/init.pp
index 7c8531d..99fcc67 100644
--- a/puppet/modules/iegreview/manifests/init.pp
+++ b/puppet/modules/iegreview/manifests/init.pp
@@ -54,6 +54,10 @@
directory => $deploy_dir,
}
+ service::gitupdate { 'iegreview':
+ dir => $deploy_dir,
+ }
+
# Create an application database
mysql::db { $db_name:
ensure => present,
diff --git a/puppet/modules/mediawiki/manifests/init.pp
b/puppet/modules/mediawiki/manifests/init.pp
index 57487cd..b4c8bfd 100644
--- a/puppet/modules/mediawiki/manifests/init.pp
+++ b/puppet/modules/mediawiki/manifests/init.pp
@@ -73,6 +73,8 @@
require ::php
require ::hhvm
+ require ::service
+
include ::mediawiki::apache
include ::mediawiki::jobrunner
include ::mediawiki::multiwiki
diff --git a/puppet/modules/mediawiki/manifests/jobrunner.pp
b/puppet/modules/mediawiki/manifests/jobrunner.pp
index 056e99c..64a5d9b 100644
--- a/puppet/modules/mediawiki/manifests/jobrunner.pp
+++ b/puppet/modules/mediawiki/manifests/jobrunner.pp
@@ -22,6 +22,11 @@
default => 'running',
}
+ $restart = $enable ? {
+ false => false,
+ default => true,
+ }
+
git::clone { 'mediawiki/services/jobrunner':
directory => $dir,
before => Service['jobrunner'],
@@ -80,6 +85,17 @@
mode => '0444',
}
+ service::gitupdate { 'jobrunner':
+ dir => $dir,
+ restart => $restart,
+ }
+
+ service::gitupdate { 'jobchron':
+ dir => $dir,
+ pull => false,
+ restart => $restart,
+ }
+
service { 'jobrunner':
enable => $enable,
ensure => $ensure,
diff --git a/puppet/modules/mediawiki/manifests/parsoid.pp
b/puppet/modules/mediawiki/manifests/parsoid.pp
index 6a491ca..5b7ff54 100644
--- a/puppet/modules/mediawiki/manifests/parsoid.pp
+++ b/puppet/modules/mediawiki/manifests/parsoid.pp
@@ -66,6 +66,11 @@
settings => { wgParsoidCacheServers => [], },
}
+ service::gitupdate { 'parsoid':
+ dir => $dir,
+ restart => true,
+ }
+
service { 'parsoid':
enable => true,
ensure => running,
diff --git a/puppet/modules/mediawiki/templates/run-git-update.erb
b/puppet/modules/mediawiki/templates/run-git-update.erb
index 381914b..10fb7bb 100644
--- a/puppet/modules/mediawiki/templates/run-git-update.erb
+++ b/puppet/modules/mediawiki/templates/run-git-update.erb
@@ -10,40 +10,44 @@
UPDATE_ERRORS=( )
function pull {
- git_cmd=git
- # /srv dirs are all root, so use sudo in that case
- [[ -O . ]] || git_cmd="sudo git"
- echo -e "\e[36m==> Updating $(pwd) ...\e[0m"
- branch=$(expr $($git_cmd symbolic-ref HEAD) : 'refs/heads/\(.*\)')
+ print_empty_line=''
+ if [[ $# -eq 0 ]]; then
+ echo -e "\e[36m==> Updating $(pwd) ...\e[0m"
+ print_empty_line='yes'
+ else
+ echo "[*] Updating repo in $(pwd) ..."
+ fi
+ # some srv dirs may be still under root, chown them
+ find . -uid 0 -exec sudo chown -R <%=
"#{scope['::share_owner']}:#{scope['::share_group']}" %> {} \;
+ branch=$(expr $(git symbolic-ref HEAD) : 'refs/heads/\(.*\)')
if [[ -z "$branch" ]]; then
- $git_cmd checkout master
- branch=$(expr $($git_cmd symbolic-ref HEAD) : 'refs/heads/\(.*\)')
+ git checkout master
+ branch=$(expr $(git symbolic-ref HEAD) : 'refs/heads/\(.*\)')
fi
err=''
if [[ -n "$branch" ]]; then
- remote=$($git_cmd config branch.${branch}.remote)
+ remote=$(git config branch.${branch}.remote)
if [[ -n "$remote" ]]; then
- url=$($git_cmd config --get remote.${remote}.url)
- if [[ $url == ssh://* ]]; then
- # Convert remote git url from ssh:// to anonymous https://
- tempurl=$(echo $url | sed -e 's!ssh://[^@]\+@!https://!g' -e
's!:29418!/r!g')
- $git_cmd pull --rebase $tempurl
- else
- $git_cmd pull --rebase
- fi
+ url=$(git config --get remote.${remote}.url)
+ if [[ $url == ssh://* ]]; then
+ # Convert remote git url from ssh:// to anonymous https://
+ tempurl=$(echo $url | sed -e 's!ssh://[^@]\+@!https://!g' -e
's!:29418!/r!g')
+ git pull --rebase $tempurl
+ else
+ git pull --rebase
+ fi
+ if [[ $? -ne 0 ]]; then
+ # If we didn't successfully update (possibly because we're on
+ # a local branch), leave the submodules alone.
+ err="GIT PULL failed in $(pwd) for branch '$branch'"
+ else
+ git submodule update --init --recursive
if [[ $? -ne 0 ]]; then
- # If we didn't successfully update (possibly because we're on
- # a local branch), leave the submodules alone.
- err="GIT PULL failed in $(pwd) for branch '$branch'"
- else
- $git_cmd submodule update --init --recursive
- if [[ $? -ne 0 ]]; then
- err="GIT SUBMODULE UPDATE failed in $(pwd) for branch '$branch'"
- fi
+ err="GIT SUBMODULE UPDATE failed in $(pwd) for branch '$branch'"
fi
-
+ fi
else
- err="Remote not found in $(pwd) for branch '$branch'"
+ err="Remote not found in $(pwd) for branch '$branch'"
fi
else
err="Unable to get current branch in $(pwd)"
@@ -53,15 +57,27 @@
echo -e "\e[1;31m ***** $err\e[0m"
UPDATE_ERRORS+=("$err")
fi
- echo
+
+ [[ -n "${print_empty_line}" ]] && echo
+
}
-for srvdir in $(find <%= scope['::mwv::services_dir'] %> -maxdepth 2 -type d
-name .git -printf "%h\n"); do
- cd "${srvdir}"
- pull
- if ! git ls-files node_modules --error-unmatch > /dev/null 2>&1 && [[ -f
package.json ]]; then
- npm update
+for srvconf in <%= scope['::service::conf_dir'] %>/*.conf; do
+ source ${srvconf}
+ echo -e "\e[36m==> Updating ${SERVICE_NAME} ...\e[0m"
+ if [[ -n "${NEED_CHDIR}" && -d ${SERVICE_DIR} ]]; then
+ cd "${SERVICE_DIR}"
+ [[ -n "${DO_PULL}" ]] && pull no_title
+ if [[ -n ${UPDATE_CMD} ]]; then
+ echo "[*] Updating dependencies for ${SERVICE_NAME} ..."
+ ${UPDATE_CMD}
+ fi
fi
+ if [[ -n "${DO_RESTART}" ]]; then
+ echo "[*] Restarting ${SERVICE_NAME} ..."
+ sudo service ${SERVICE_NAME} restart
+ fi
+ echo
done
cd "$MW_INSTALL_PATH"
diff --git a/puppet/modules/phabricator/manifests/init.pp
b/puppet/modules/phabricator/manifests/init.pp
index 36a2603..f1f153d 100644
--- a/puppet/modules/phabricator/manifests/init.pp
+++ b/puppet/modules/phabricator/manifests/init.pp
@@ -29,6 +29,10 @@
require => Class['::arcanist'],
}
+ service::gitupdate { 'phd':
+ dir => "${deploy_dir}/phabricator",
+ }
+
# Add our vhost
apache::site { $vhost_name:
ensure => present,
@@ -36,9 +40,19 @@
require => Class['::apache::mod::rewrite'],
}
+ phabricator::config { 'mysql.host':
+ value => '127.0.0.1',
+ require => Class['::mysql'],
+ }
+
+ phabricator::config { 'mysql.port':
+ value => 3306,
+ require => Phabricator::Config['mysql.host'],
+ }
+
phabricator::config { 'mysql.pass':
value => $::mysql::root_password,
- require => Class['::mysql'],
+ require => Phabricator::Config['mysql.port'],
}
phabricator::config { 'phabricator.base-uri':
diff --git a/puppet/modules/role/manifests/phragile.pp
b/puppet/modules/role/manifests/phragile.pp
index c3e27f9..979aea0 100644
--- a/puppet/modules/role/manifests/phragile.pp
+++ b/puppet/modules/role/manifests/phragile.pp
@@ -21,6 +21,12 @@
remote => 'https://github.com/wmde/phragile.git',
}
+ service::gitupdate { 'phragile':
+ dir => $install_dir,
+ type => 'php',
+ update => true,
+ }
+
file { "${install_dir}/.env":
content => template('role/phragile/env.erb'),
require => Git::Clone['https://github.com/wmde/phragile.git'],
diff --git a/puppet/modules/scholarships/manifests/init.pp
b/puppet/modules/scholarships/manifests/init.pp
index 36e3b35..242dc0b 100644
--- a/puppet/modules/scholarships/manifests/init.pp
+++ b/puppet/modules/scholarships/manifests/init.pp
@@ -46,6 +46,10 @@
directory => $deploy_dir,
}
+ service::gitupdate { 'scholarships':
+ dir => $deploy_dir,
+ }
+
# Create an application database
mysql::db { $db_name:
ensure => present,
diff --git a/puppet/modules/service/manifests/gitupdate.pp
b/puppet/modules/service/manifests/gitupdate.pp
new file mode 100644
index 0000000..4f07d35
--- /dev/null
+++ b/puppet/modules/service/manifests/gitupdate.pp
@@ -0,0 +1,33 @@
+define service::gitupdate(
+ $dir = undef,
+ $type = undef,
+ $pull = true,
+ $update = false,
+ $restart = false,
+) {
+
+ require ::service
+
+ # descern the update command to use
+ $up_cmd = $type ? {
+ 'php' => 'composer update --no-interaction --optimize-autoloader',
+ 'nodejs' => 'npm update --no-bin-links',
+ default => 'invalid'
+ }
+ if $update and $up_cmd == 'invalid' {
+ fail("Invalid service type ${type} given, valid values are php,
nodejs")
+ }
+
+ $srv_dir = $dir ? {
+ undef => "${::service::root_dir}/${title}",
+ default => $dir
+ }
+
+ file { "${::service::conf_dir}/${title}.conf":
+ ensure => present,
+ content => template('service/gitupdate.conf.erb'),
+ owner => $::share_owner,
+ group => $::share_group,
+ }
+
+}
diff --git a/puppet/modules/service/manifests/init.pp
b/puppet/modules/service/manifests/init.pp
index eee5f1d..f08f701 100644
--- a/puppet/modules/service/manifests/init.pp
+++ b/puppet/modules/service/manifests/init.pp
@@ -7,6 +7,9 @@
# [*root_dir*]
# The directory where to install WMF services.
#
+# [*conf_dir*]
+# The directory containing the configuration files managing service updates.
+#
# [*log_dir*]
# The directory where the logs should be stored.
#
@@ -16,9 +19,16 @@
#
class service (
$root_dir,
+ $conf_dir,
$log_dir,
$log_level,
) {
- # no-op, just config for now
+
+ file { $conf_dir:
+ ensure => directory,
+ owner => $::share_owner,
+ group => $::share_group,
+ }
+
}
diff --git a/puppet/modules/service/manifests/node.pp
b/puppet/modules/service/manifests/node.pp
index 6c1a61d..4c97bff 100644
--- a/puppet/modules/service/manifests/node.pp
+++ b/puppet/modules/service/manifests/node.pp
@@ -135,6 +135,13 @@
notify => Service[$title],
}
+ # schedule the service for git-updates via vagrant git-update
+ service::gitupdate { $title:
+ type => 'nodejs',
+ update => true,
+ restart => true,
+ }
+
# the service definition
service { $title:
ensure => running,
diff --git a/puppet/modules/service/templates/gitupdate.conf.erb
b/puppet/modules/service/templates/gitupdate.conf.erb
new file mode 100644
index 0000000..5913ce1
--- /dev/null
+++ b/puppet/modules/service/templates/gitupdate.conf.erb
@@ -0,0 +1,6 @@
+SERVICE_NAME=<%= @title %>
+SERVICE_DIR=<%= @srv_dir %>
+NEED_CHDIR=<%= if @pull || @update then 'yes' else '' end %>
+DO_PULL=<%= if @pull then 'yes' else '' end %>
+UPDATE_CMD=<%= if @update then "'#{@up_cmd}'" else '' end %>
+DO_RESTART=<%= if @restart then 'yes' else '' end %>
diff --git a/puppet/modules/smashpig/manifests/init.pp
b/puppet/modules/smashpig/manifests/init.pp
index 2b04a1d..affbfe9 100644
--- a/puppet/modules/smashpig/manifests/init.pp
+++ b/puppet/modules/smashpig/manifests/init.pp
@@ -14,6 +14,12 @@
directory => $dir,
}
+ service::gitupdate { 'smashpig':
+ dir => $dir,
+ type => 'php',
+ update => true,
+ }
+
file { "${dir}/config.php":
content => template('smashpig/config.php.erb'),
require => [
--
To view, visit https://gerrit.wikimedia.org/r/219136
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I7ec1d2eaa3a003d4c3bfb80a29fa11fb64506407
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/vagrant
Gerrit-Branch: master
Gerrit-Owner: Mobrovac <mobro...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
