Ryan Lane has uploaded a new change for review.
https://gerrit.wikimedia.org/r/110239
Change subject: Deployment module changes for trebuchet-trigger
......................................................................
Deployment module changes for trebuchet-trigger
Change-Id: I4f736f833e85498acddda60f4ea3a8797f44672b
---
M manifests/role/deployment.pp
D modules/deployment/files/git-deploy/dependencies/l10nupdate-quick
D modules/deployment/files/git-deploy/hooks/depends.py
D modules/deployment/files/git-deploy/hooks/deploylib.py
D modules/deployment/files/git-deploy/hooks/shared.py
M modules/deployment/files/modules/deploy.py
M modules/deployment/manifests/deployment_server.pp
D modules/deployment/templates/git-deploy/git-deploy.conf.erb
D modules/deployment/templates/git-deploy/gitconfig.erb
D modules/deployment/templates/git-deploy/gitignore.erb
10 files changed, 17 insertions(+), 508 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/39/110239/1
diff --git a/manifests/role/deployment.pp b/manifests/role/deployment.pp
index 30e0967..7f483a6 100644
--- a/manifests/role/deployment.pp
+++ b/manifests/role/deployment.pp
@@ -227,7 +227,7 @@
vhost_name => "10.64.0.196",
port => 80,
docroot => "/srv/deployment",
- docroot_owner => "sartoris",
+ docroot_owner => "trebuchet",
docroot_group => "wikidev",
docroot_dir_allows => ["10.0.0.0/16","10.64.0.0/16","208.80.152.0/22"],
serveradmin => "n...@wikimedia.org",
@@ -260,7 +260,7 @@
vhost_name => "10.4.0.58",
port => 80,
docroot => "/srv/deployment",
- docroot_owner => "sartoris",
+ docroot_owner => "trebuchet",
docroot_group => "project-deployment-prep",
docroot_dir_allows => ["10.4.0.0/16"],
serveradmin => "n...@wikimedia.org",
diff --git a/modules/deployment/files/git-deploy/dependencies/l10nupdate-quick
b/modules/deployment/files/git-deploy/dependencies/l10nupdate-quick
deleted file mode 100755
index 45e17ae..0000000
--- a/modules/deployment/files/git-deploy/dependencies/l10nupdate-quick
+++ /dev/null
@@ -1,123 +0,0 @@
-#!/bin/bash
-
-set -e
-
-BINDIR=/usr/local/bin
-
-. /usr/local/lib/mw-deployment-vars.sh
-
-umask 0002
-echo "Starting l10nupdate-quick at `date`."
-
-mwVerDbSets=$($BINDIR/mwversionsinuse --withdb)
-if [ -z "$mwVerDbSets" ]; then
- echo "Obtaining MediaWiki version list FAILED"
- exit 1
-fi
-
-# Update l10n cache
-for i in ${mwVerDbSets[@]}
-do
- mwVerNum=${i%=*}
- mwDbName=${i#*=}
- slot=`basename "$(readlink -e $MW_COMMON/l10n-$mwVerNum)"`
-
- if [ ! -z "$1" -a "$1" != "$slot" ]
- then
- continue
- fi
-
- if [ ! -d "$MW_COMMON/l10n-$mwVerNum" ]
- then
- echo "Update for $mwVerNum failed: $MW_COMMON/l10n-$mwVerNum
does not exist"
- continue
- fi
-
- cd $MW_COMMON/l10n-$mwVerNum
-
- git deploy start
- set +e
- FAILMSG=""
-
- trap "{
- echo Cleaning up after signal
- git clean -d -f
- git reset --hard
- git deploy abort
- exit 255
- }" SIGINT SIGTERM
-
- if [ ! -d "$MW_COMMON/l10n-$mwVerNum/cache" ]
- then
- mkdir $MW_COMMON/l10n-$mwVerNum/cache
- fi
-
- if [ ! -e "$MW_COMMON/l10n-$mwVerNum/ExtensionMessages.php" ]
- then
- touch $MW_COMMON/l10n-$mwVerNum/ExtensionMessages.php
- fi
-
- if [ ! -e "$MW_COMMON/l10n-$mwVerNum/cache/l10n_cache-en.cdb" ]
- then
- echo "Building initial localisation cache for $mwVerNum (on
$mwDbName)"
- if $BINDIR/mwscript rebuildLocalisationCache.php
--wiki="$mwDbName" \
- --outdir=$MW_COMMON/l10n-$mwVerNum/cache \
- --threads=12
- then
- true
- else
- FAILMSG="Localisation cache build failed"
- fi
- fi
-
- if [ -z "$FAILMSG" ]
- then
- echo "Updating ExtensionMessages.php for $mwVerNum (on
$mwDbName)"
- if $BINDIR/mwscript mergeMessageFileList.php --wiki="$mwDbName"
\
- --list-file=$MW_COMMON/wmf-config/extension-list \
- --output=$MW_COMMON/l10n-$mwVerNum/ExtensionMessages.php
- then
- true
- else
- FAILMSG="ExtensionMessages update failed"
- fi
- fi
-
- if [ -z "$FAILMSG" ]
- then
- echo "Rebuilding localisation cache for $mwVerNum (on
$mwDbName)"
- if $BINDIR/mwscript rebuildLocalisationCache.php
--wiki="$mwDbName" \
- --outdir=$MW_COMMON/l10n-$mwVerNum/cache \
- --threads=12
- then
- true
- else
- FAILMSG="Localisation cache rebuild failed"
- fi
- fi
-
- if [ -z "$FAILMSG" ]
- then
- git add ExtensionMessages.php cache
-
- if git status --porcelain | grep -q '^[MADRC]'
- then
- echo "Deploying change to localisation for $mwVerNum
(on $mwDbName)"
- git commit -m "Update localisation cache for $mwVerNum"
- git deploy sync
- else
- echo "No change to localisation for $mwVerNum (on
$mwDbName)"
- git clean -d -f
- git reset --hard
- git deploy abort
- fi
- else
- echo "$FAILMSG"
- git clean -d -f
- git reset --hard
- git deploy abort
- fi
-
- trap "" SIGINT SIGTERM
- set -e
-done
diff --git a/modules/deployment/files/git-deploy/hooks/depends.py
b/modules/deployment/files/git-deploy/hooks/depends.py
deleted file mode 100644
index aecd43a..0000000
--- a/modules/deployment/files/git-deploy/hooks/depends.py
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/python
-
-import os
-import deploylib
-
-
-def main():
- prefix = os.environ['DEPLOY_ROLLOUT_PREFIX']
- tag = os.environ['DEPLOY_ROLLOUT_TAG']
- force = os.environ['DEPLOY_FORCE']
- #TODO: Use this message to notify IRC
- #msg = os.environ['DEPLOY_DEPLOY_TEXT']
-
- prefixlib = deploylib.DeployLib(prefix)
- prefixlib.update_repos(tag)
- # In general, for dependent repos, the parent repo is handling
- # fetch and checkout. Some dependent repos also update outside
- # of their parent repos. If the repo forces a sync, then we should
- # handle it.
- if force:
- prefixlib.fetch()
- prefixlib.checkout("True")
-
-if __name__ == "__main__":
- main()
diff --git a/modules/deployment/files/git-deploy/hooks/deploylib.py
b/modules/deployment/files/git-deploy/hooks/deploylib.py
deleted file mode 100755
index b82f136..0000000
--- a/modules/deployment/files/git-deploy/hooks/deploylib.py
+++ /dev/null
@@ -1,146 +0,0 @@
-#!/usr/bin/python
-
-import os
-import subprocess
-import json
-
-
-class DeployLib(object):
-
- __config = {}
-
- def __init__(self, prefix):
- self.__fetch_config(prefix)
-
- def __fetch_config(self, prefix):
- print "Running: sudo salt-call -l quiet --out json pillar.data"
- p = subprocess.Popen("sudo salt-call -l quiet --out json pillar.data",
- shell=True, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- out = p.communicate()[0]
- try:
- pillar = json.loads(out)
- if 'local' in pillar:
- pillar = pillar['local']
- try:
- repo_config = pillar['repo_config'][prefix]
- parent_dir = pillar['deployment_config']['parent_dir']
- except KeyError:
- print ("Missing configuration for repo. "
- "Have you added it in puppet? Exiting.")
- return False
- options = {'location': '{0}/{1}'.format(parent_dir,
- prefix),
- 'automated': False,
- 'checkout_submodules': False,
- 'dependencies': {}}
- for option, default in options.items():
- try:
- self.__config[option] = repo_config[option]
- except KeyError:
- self.__config[option] = default
- self.__config['prefix'] = prefix
- return True
- except ValueError:
- print ("JSON data wasn't loaded from the pillar call. "
- "git-deploy can't configure itself. Exiting.")
- return False
-
- def get_config(self):
- return self.__config
-
- def update_repos(self, tag):
- repodir = self.__config['location']
- checkout_submodules = self.__config['checkout_submodules']
-
- # Ensure the fetch will work for the repo
- p = subprocess.Popen('git update-server-info',
- cwd=repodir + '/.git/', shell=True,
- stderr=subprocess.PIPE)
- err = p.communicate()[0]
- if err:
- print err
- # Ensure the fetch will work for the submodules
- if checkout_submodules:
- cmd = 'git submodule foreach --recursive "git tag {0}"'.format(tag)
- p = subprocess.Popen(cmd, cwd=repodir, shell=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- out = p.communicate()[0]
- p = subprocess.Popen('git submodule foreach --recursive '
- '"submodule-update-server-info"',
- cwd=repodir, shell=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- out = p.communicate()[0]
-
- def run_dependencies(self):
- # Ensure repos we depend on are handled
- dependencies = self.__config['dependencies']
- for dependency, script in dependencies.items():
- dependency_script = ('/var/lib/git-deploy/dependencies/%s' %
- (script))
- if os.path.exists(dependency_script):
- dependency_script = (dependency_script + " %s %s" %
- (dependency, self.__config['prefix']))
- p = subprocess.Popen(dependency_script, shell=True,
- stderr=subprocess.PIPE)
- out = p.communicate()[0]
- print out
- else:
- print ("Error: script for dependency '%s' is missing. "
- "Have you added it in puppet? Exiting." %
- dependency_script)
- return 1
-
- def fetch(self):
- prefix = self.__config['prefix']
- print ("Running: sudo salt-call -l quiet publish.runner "
- "deploy.fetch '%s'" % (prefix))
- p = subprocess.Popen("sudo salt-call -l quiet publish.runner "
- "deploy.fetch '%s'" % (prefix),
- shell=True,
- stdout=subprocess.PIPE)
- out = p.communicate()[0]
-
- def checkout(self, force):
- prefix = self.__config['prefix']
- print ("Running: sudo salt-call -l quiet publish.runner "
- "deploy.checkout '%s,%s'" % (prefix, force))
- p = subprocess.Popen("sudo salt-call -l quiet publish.runner "
- "deploy.checkout '%s,%s'" % (prefix, force),
- shell=True, stdout=subprocess.PIPE)
- out = p.communicate()[0]
-
- def ask(self, stage, force=False):
- prefix = self.__config['prefix']
- if stage == "fetch":
- check = "/usr/local/bin/deploy-info --repo=%s --fetch"
- elif stage == "checkout":
- check = "/usr/local/bin/deploy-info --repo=%s"
- p = subprocess.Popen(check % (prefix), shell=True,
- stdout=subprocess.PIPE)
- out = p.communicate()[0]
- print out
- while True:
- answer = raw_input("Continue? ([d]etailed/[C]oncise report,"
- "[y]es,[n]o,[r]etry): ")
- if not answer or answer == "c" or answer == "C":
- p = subprocess.Popen(check % (prefix), shell=True,
- stdout=subprocess.PIPE)
- out = p.communicate()[0]
- print out
- elif answer == "d" or answer == "D":
- p = subprocess.Popen(check % (prefix) + " --detailed",
- shell=True, stdout=subprocess.PIPE)
- out = p.communicate()[0]
- print out
- elif answer == "Y" or answer == "y":
- return True
- elif answer == "N" or answer == "n":
- return False
- elif answer == "R" or answer == "r":
- if stage == "fetch":
- self.fetch()
- if stage == "checkout":
- self.checkout(force)
diff --git a/modules/deployment/files/git-deploy/hooks/shared.py
b/modules/deployment/files/git-deploy/hooks/shared.py
deleted file mode 100644
index e1d2ed8..0000000
--- a/modules/deployment/files/git-deploy/hooks/shared.py
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/usr/bin/python
-
-import os
-import redis
-import getpass
-import deploylib
-
-
-def main():
- prefix = os.environ['DEPLOY_ROLLOUT_PREFIX']
- tag = os.environ['DEPLOY_ROLLOUT_TAG']
- force = os.environ['DEPLOY_FORCE']
- if force:
- force = "True"
- else:
- force = "False"
-
- prefixlib = deploylib.DeployLib(prefix)
- config = prefixlib.get_config()
- if not config:
- return 1
-
- if config['automated']:
- log = 'automated deployment'
- else:
- log = raw_input("Log message: ")
- serv = redis.Redis(host='localhost', port=6379, db=0)
- serv.rpush("deploy:log", "!log {0} started synchronizing "
- "{1} '{2}'".format(getpass.getuser(), tag, log))
-
- prefixlib.update_repos(tag)
- prefixlib.fetch()
- if not prefixlib.ask('fetch'):
- return 1
- prefixlib.run_dependencies()
- prefixlib.checkout(force)
- if not prefixlib.ask('checkout', force):
- return 1
-
- serv.rpush("deploy:log", "!log {0} synchronized "
- "{1} '{2}'".format(getpass.getuser(), tag, log))
-
-if __name__ == "__main__":
- main()
diff --git a/modules/deployment/files/modules/deploy.py
b/modules/deployment/files/modules/deploy.py
index 6ec43c9..7362ee2 100644
--- a/modules/deployment/files/modules/deploy.py
+++ b/modules/deployment/files/modules/deploy.py
@@ -90,26 +90,13 @@
def deployment_server_init():
- serv = _get_redis_serv()
is_deployment_server = __grains__.get('deployment_server')
- hook_dir = __grains__.get('deployment_global_hook_dir')
if not is_deployment_server:
return 0
deploy_user = __grains__.get('deployment_repo_user')
repo_config = __pillar__.get('repo_config')
for repo in repo_config:
config = get_config(repo)
- repo_sync_dir = '{0}/sync/{1}'.format(hook_dir, os.path.dirname(repo))
- sync_link = '{0}/{1}.sync'.format(repo_sync_dir,
- os.path.basename(repo))
- # Create repo sync dir
- if not __salt__['file.directory_exists'](repo_sync_dir):
- __salt__['file.mkdir'](repo_sync_dir)
- # Create repo sync script link
- if not __salt__['file.file_exists'](sync_link):
- sync_script = '{0}/sync/{1}'.format(hook_dir,
- config['sync_script'])
- __salt__['file.symlink'](sync_script, sync_link)
# Clone repo from upstream or init repo with no upstream
if not __salt__['file.directory_exists'](config['location'] + '/.git'):
if config['upstream']:
@@ -124,7 +111,7 @@
# git clone does ignores umask and does explicit mkdir with 755
__salt__['file.set_mode'](config['location'], 2775)
# Set the repo name in the repo's config
- cmd = 'git config deploy.tag-prefix %s' % repo
+ cmd = 'git config deploy.repo-name %s' % repo
status = __salt__['cmd.retcode'](cmd, cwd=config['location'],
runas=deploy_user, umask=002)
if status != 0:
diff --git a/modules/deployment/manifests/deployment_server.pp
b/modules/deployment/manifests/deployment_server.pp
index 69ecb12..67a1b87 100644
--- a/modules/deployment/manifests/deployment_server.pp
+++ b/modules/deployment/manifests/deployment_server.pp
@@ -1,25 +1,4 @@
-class deployment::deployment_server(
- $deployment_conffile='/etc/git-deploy/git-deploy.conf',
- $deployment_ignorefile='/etc/git-deploy/gitignore',
- $deployment_ignores=['.deploy'],
- $deployment_restrict_umask='002',
- $deployment_block_file='/etc/ROLLOUTS_BLOCKED',
- $deployment_support_email='',
- $deployment_repo_name_detection='dot-git-parent-dir',
- $deployment_announce_email='',
- $deployment_send_mail_on_sync=false,
- $deployment_send_mail_on_revert=false,
- $deployment_log_directory='/var/log/git-deploy',
- $deployment_log_timing_data=false,
- $deployment_git_deploy_dir='/var/lib/git-deploy',
- $deployment_per_repo_config={},
- $deployer_groups=[]
- ) {
- if ! defined(Package['git-deploy']){
- package { 'git-deploy':
- ensure => present;
- }
- }
+class deployment::deployment_server($deployer_groups=[]) {
if ! defined(Package['git-core']){
package { 'git-core':
ensure => present;
@@ -30,85 +9,16 @@
ensure => present;
}
}
-
- $deployment_global_hook_dir = "${deployment_git_deploy_dir}/hooks"
- $deployment_dependencies_dir = "${deployment_git_deploy_dir}/dependencies"
- file { $deployment_global_hook_dir:
- ensure => directory,
- mode => '0555',
- owner => 'root',
- group => 'root',
+ if ! defined(Package['python-git']){
+ package { 'python-git':
+ ensure => present;
+ }
+ }
+ package { 'trebuchet-trigger':
+ ensure => present;
}
- file { $deployment_dependencies_dir:
- ensure => directory,
- mode => '0555',
- owner => 'root',
- group => 'root',
- }
-
- file { "${$deployment_global_hook_dir}/sync":
- ensure => directory,
- mode => '0555',
- owner => 'root',
- group => 'root',
- require => [File[$deployment_global_hook_dir]],
- }
-
- file { "${$deployment_global_hook_dir}/sync/deploylib.py":
- source => 'puppet:///deployment/git-deploy/hooks/deploylib.py',
- mode => '0555',
- owner => 'root',
- group => 'root',
- require => [File["${$deployment_global_hook_dir}/sync"]],
- }
-
- file { "${$deployment_global_hook_dir}/sync/shared.py":
- source => 'puppet:///deployment/git-deploy/hooks/shared.py',
- mode => '0555',
- owner => 'root',
- group => 'root',
- require => [File["${$deployment_global_hook_dir}/sync"]],
- }
-
- file { "${$deployment_global_hook_dir}/sync/depends.py":
- source => 'puppet:///deployment/git-deploy/hooks/depends.py',
- mode => '0555',
- owner => 'root',
- group => 'root',
- require => [File["${$deployment_global_hook_dir}/sync"]],
- }
-
- file { "${$deployment_dependencies_dir}/l10n":
- source =>
'puppet:///deployment/git-deploy/dependencies/l10nupdate-quick',
- mode => '0555',
- owner => 'root',
- group => 'root',
- require => [File[$deployment_dependencies_dir]],
- }
-
- file { '/etc/gitconfig':
- content => template('deployment/git-deploy/gitconfig.erb'),
- mode => '0444',
- owner => 'root',
- group => 'root',
- require => [Package['git-core']],
- }
-
- file { $deployment_conffile:
- content => template('deployment/git-deploy/git-deploy.conf.erb'),
- mode => '0444',
- owner => 'root',
- group => 'root',
- }
-
- file { $deployment_ignorefile:
- content => template('deployment/git-deploy/gitignore.erb'),
- mode => '0444',
- owner => 'root',
- group => 'root',
- }
-
+ # Remove when added to trigger
file { '/usr/local/bin/deploy-info':
owner => 'root',
group => 'root',
@@ -117,6 +27,7 @@
require => [Package['python-redis']],
}
+ # Remove when added to trigger
file { '/usr/local/bin/service-restart':
owner => 'root',
group => 'root',
@@ -124,6 +35,7 @@
source => 'puppet:///deployment/git-deploy/utils/service-restart',
}
+ # Remove when added to trigger
file { '/usr/local/bin/submodule-update-server-info':
owner => 'root',
group => 'root',
@@ -137,20 +49,14 @@
replace => true,
}
- salt::grain { 'deployment_global_hook_dir':
- grain => 'deployment_global_hook_dir',
- value => $deployment_global_hook_dir,
- replace => true,
- }
-
salt::grain { 'deployment_repo_user':
grain => 'deployment_repo_user',
- value => 'sartoris',
+ value => 'trebuchet',
replace => true,
}
- generic::systemuser { 'sartoris':
- name => 'sartoris',
+ generic::systemuser { 'trebuchet':
+ name => 'trebuchet',
shell => '/bin/false',
home => '/nonexistent',
groups => $deployer_groups,
diff --git a/modules/deployment/templates/git-deploy/git-deploy.conf.erb
b/modules/deployment/templates/git-deploy/git-deploy.conf.erb
deleted file mode 100644
index d9512a2..0000000
--- a/modules/deployment/templates/git-deploy/git-deploy.conf.erb
+++ /dev/null
@@ -1,39 +0,0 @@
-;; Global options
-[deploy]
- ;; Force users to have this umask
- restrict-umask = <%= deployment_restrict_umask %>
-
- ;; If this file exists all rollouts are blocked
- block-file = <%= deployment_block_file %>
-
- ;; E-Mail addresses to complain to when stuff goes wrong
- <% if deployment_support_email %>support-email = <%=
deployment_support_email %><% else %>;;support-email = ad...@example.org<% end
%>
-
- ;; What strategy should we use to detect the repo name?
- repo-name-detection = <%= deployment_repo_name_detection %>
-
- ;; Where should the mail configured below go?
- <% if deployment_announce_email %>announce-mail = <%=
deployment_announce_email %><% else %>;;announce-mail = admin@examp
-le.org<% end %>
-
- ;; When should we send an E-Mail?
- send-mail-on-sync = <%= deployment_send_mail_on_sync %>
- send-mail-on-revert = <%= deployment_send_mail_on_revert %>
-
- ;; Where to store the timing information
- log-directory = <%= deployment_log_directory %>
-
- ;; We want timing information
- log-timing-data = <%= deployment_log_timing_data %>
-
- ;; Global hook dir
- <% if deployment_global_hook_dir %>hook-dir = <%=
deployment_global_hook_dir %><% else %>;;hoor-dir = /var/lib/git-deploy/hooks<%
end %>
-
-;; Per-repo options, keys here override equivalent keys in the
-;; global options
-<% if deployment_per_repo_config %><% deployment_per_repo_config.each do
|repo, options| %>
-[deploy "repository <%= repo %>"]
-<% options.each do |option, value| %>
- <%= option %> = <%= value %>
-<% end %><% end %>
-<% end %>
diff --git a/modules/deployment/templates/git-deploy/gitconfig.erb
b/modules/deployment/templates/git-deploy/gitconfig.erb
deleted file mode 100644
index 48bd6c8..0000000
--- a/modules/deployment/templates/git-deploy/gitconfig.erb
+++ /dev/null
@@ -1,4 +0,0 @@
-[deploy]
- config-file = <%= deployment_conffile %>
-[core]
- excludesfile = <%= deployment_ignorefile %>
diff --git a/modules/deployment/templates/git-deploy/gitignore.erb
b/modules/deployment/templates/git-deploy/gitignore.erb
deleted file mode 100644
index db6ca81..0000000
--- a/modules/deployment/templates/git-deploy/gitignore.erb
+++ /dev/null
@@ -1,3 +0,0 @@
-<% deployment_ignores.each do |deployment_ignore| %>
-<%= deployment_ignore %>
-<% end %>
--
To view, visit https://gerrit.wikimedia.org/r/110239
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I4f736f833e85498acddda60f4ea3a8797f44672b
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ryan Lane <rl...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
