Muehlenhoff has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/379189 )
Change subject: Remove deployment::salt_master/role::deployment::salt_masters and related files ...................................................................... Remove deployment::salt_master/role::deployment::salt_masters and related files Change-Id: Ide7b78e46617338ff9e044a6939519afd40d404e --- M hieradata/labs/deployment-prep/common.yaml M manifests/site.pp D modules/deployment/files/modules/deploy.py D modules/deployment/files/returners/deploy_redis.py D modules/deployment/files/runners/deploy.py D modules/deployment/files/states/deploy/sync_all.sls D modules/deployment/files/states/top.sls D modules/deployment/manifests/salt_master.pp D modules/deployment/templates/pillars/top.sls.erb D modules/role/manifests/deployment/salt_masters.pp 10 files changed, 3 insertions(+), 1,137 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/89/379189/1 diff --git a/hieradata/labs/deployment-prep/common.yaml b/hieradata/labs/deployment-prep/common.yaml index 8b89570..19fa991 100644 --- a/hieradata/labs/deployment-prep/common.yaml +++ b/hieradata/labs/deployment-prep/common.yaml @@ -173,7 +173,6 @@ "misc::syslog-server::basepath": /data/project/syslog "profile::cxserver::apertium_uri": http://apertium-beta.wmflabs.org role::deployment::mediawiki::key_fingerprint: f0:54:06:fa:17:27:97:a2:cc:69:a0:a7:df:4c:0a:e3 -"role::deployment::salt_masters::deployment_server": deployment-tin.deployment-prep.eqiad.wmflabs "hhvm::extra::fcgi": hhvm: pcre_cache_type: lru diff --git a/manifests/site.pp b/manifests/site.pp index 450c265..5604192 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -1238,8 +1238,7 @@ node 'labcontrol1001.wikimedia.org' { role(wmcs::openstack::main::control, - salt::masters::labs, - deployment::salt_masters) + salt::masters::labs) include ::base::firewall include ::standard @@ -1253,8 +1252,7 @@ # 'keystone endpoint-create' and 'keystone endpoint-delete.' node 'labcontrol1002.wikimedia.org' { role(wmcs::openstack::main::control, - salt::masters::labs, - deployment::salt_masters) + salt::masters::labs) include ::base::firewall include ::standard @@ -1871,7 +1869,7 @@ # cluster management (salt master, cumin master) + other management tools node 'neodymium.eqiad.wmnet' { - role(deployment::salt_masters, mgmt::drac_ilo, cluster::management) + role(mgmt::drac_ilo, cluster::management) interface::add_ip6_mapped { 'main': } } diff --git a/modules/deployment/files/modules/deploy.py b/modules/deployment/files/modules/deploy.py deleted file mode 100644 index 44db9bb..0000000 --- a/modules/deployment/files/modules/deploy.py +++ /dev/null @@ -1,801 +0,0 @@ -''' -Run git deployment commands -''' - -import redis -import time -import re -import urllib -import os -import json -import pwd -import salt - - -def _get_redis_serv(): - ''' - Return a redis server object - - :rtype: A Redis object - ''' - deployment_config = __pillar__.get('deployment_config') - deploy_redis = deployment_config['redis'] - socket_connect_timeout = deploy_redis.get('socket_connect_timeout') - if socket_connect_timeout is not None: - socket_connect_timeout = int(socket_connect_timeout) - serv = redis.Redis(host=deploy_redis['host'], - port=int(deploy_redis['port']), - db=int(deploy_redis['db']), - socket_timeout=socket_connect_timeout) - return serv - - -def _check_in(function, repo): - """ - Private function used for reporting that a function has started. - Writes to redis with basic status information. - - :param function: The function being reported on. - :type function: str - :param repo: The repository being acted on. - :type repo: str - :rtype: None - """ - serv = _get_redis_serv() - minion = __grains__.get('id') - timestamp = time.time() - # Ensure this repo exist in the set of repos - serv.sadd('deploy:repos', repo) - # Ensure this minion exists in the set of minions - serv.sadd('deploy:{0}:minions'.format(repo), minion) - minion_key = 'deploy:{0}:minions:{1}'.format(repo, minion) - if function == "deploy.fetch": - serv.hset(minion_key, 'fetch_checkin_timestamp', timestamp) - elif function == "deploy.checkout": - serv.hset(minion_key, 'checkout_checkin_timestamp', timestamp) - elif function == "deploy.restart": - serv.hset(minion_key, 'restart_checkin_timestamp', timestamp) - - -def _map_args(repo, args): - """ - Maps a set of arguments to a predefined set of values. Currently only - __REPO__ is support and will be replaced with the repository name. - - :param repo: The repo name used for mapping. - :type repo: str - :param args: An array of arguments to map. - :type args: list - :rtype: list - """ - arg_map = {'__REPO__': repo} - mapped_args = [] - for arg in args: - mapped_args.append(arg_map.get(arg, arg)) - return mapped_args - - -def get_config(repo): - """ - Fetches the configuration for this repo from the pillars and returns - a hash with the munged configuration (with defaults and helper config). - - :param repo: The specific repo for which to return config data. - :type repo: str - :rtype: hash - """ - deployment_config = __pillar__.get('deployment_config') - config = __pillar__.get('repo_config') - config = config[repo] - config.setdefault('type', 'git-http') - # location is the location on the filesystem of the repository - # shadow_location is the location on the filesystem of the shadow - # reference repository. - if 'location' in config: - location = config['location'] - shadow_location = '{0}/.{1}'.format(os.path.dirname(location), - os.path.basename(location)) - config['shadow_location'] = shadow_location - else: - location = '{0}/{1}'.format(deployment_config['parent_dir'], repo) - config['location'] = location - shadow_repo = '{0}/.{1}'.format(os.path.dirname(repo), - os.path.basename(repo)) - shadow_location = '{0}/{1}'.format(deployment_config['parent_dir'], - shadow_repo) - config['shadow_location'] = shadow_location - site = __grains__.get('site') - server = deployment_config['servers'][site] - # TODO: fetch scheme/url from implementation - if config['type'] == 'git-http': - scheme = 'http' - elif config['type'] == 'git-https': - scheme = 'https' - elif config['type'] == 'git-ssh': - scheme = 'ssh' - elif config['type'] == 'git': - scheme = 'git' - else: - scheme = 'http' - # The url of the remote on the deployment server. Target hosts will fetch - # from this during deployment. - config['url'] = '{0}://{1}/{2}'.format(scheme, server, repo) - # checkout_submodules determines whether or not this repo should - # recursively fetch and checkout submodules. - config.setdefault('checkout_submodules', False) - # If gitfat_enabled is true, git-fat will be initialized and - # git-fat pull will be run on each target as part of the checkout. - config.setdefault('gitfat_enabled', False) - # dependencies are a set of repositories that should be fetched - # and checked out before this repo. This is a deprecated feature. - config.setdefault('dependencies', {}) - # fetch_module_calls is a hash of salt modules with a list of arguments - # that will be called at the end of the fetch stage. - # TODO (ryan-lane): add a pre-fetch option - config.setdefault('fetch_module_calls', {}) - # checkout_module_calls is a hash of salt modules with a list of arguments - # that will be called at the end of the checkout stage. - # TODO (ryan-lane): add a pre-checkout option - config.setdefault('checkout_module_calls', {}) - # sync_script specifies the script that should be linked to on the - # deployment server for the perl git-deploy. This option is deprecated. - config.setdefault('sync_script', 'shared.py') - # upstream specifies the upstream url of the repository and is used - # to clone repositories on the deployment server. - config.setdefault('upstream', None) - # shadow_reference determines whether or not to make a reference clone - # of a repository on the minions during the fetch stage. This feature - # enables fetch_module_calls modules to run commands against the current - # checkout of code before it's made live. - config.setdefault('shadow_reference', False) - # service_name is the service associated with this repository and - # allows the deployment module to run service restart/stop/start/etc - # for services without allowing end-users the ability to restart all - # services on the targets. - config.setdefault('service_name', None) - # deployment_repo_group is the group that will own the repository - # after deployment_server_init. This option overrides the - # deployment_repo_group grain set for all repositories - config.setdefault('deployment_repo_group', None) - return config - - -def deployment_server_init(): - """ - Initializes a set of repositories on the deployment server. This - function will only run on the deployment server and will initialize - any repository defined in the pillar configuration. This function is - safe to call at any point. - - :rtype: int - """ - ret_status = 0 - is_deployment_server = __grains__.get('deployment_server') - if not is_deployment_server: - return ret_status - deploy_user = __grains__.get('deployment_repo_user') - deploy_group = __grains__.get('deployment_repo_group') - repo_config = __pillar__.get('repo_config') - for repo in repo_config: - config = get_config(repo) - # Clone repo from upstream or init repo with no upstream - if not __salt__['file.directory_exists'](config['location'] + '/.git'): - if config['upstream']: - cmd = '/usr/bin/git clone %s/.git %s' % (config['upstream'], - config['location']) - status = __salt__['cmd.retcode'](cmd, runas=deploy_user, - umask=002) - if status != 0: - ret_status = 1 - continue - # We don't check the checkout_submodules config flag here - # on purpose. The deployment server should always have a - # fully recursive clone and minions should decide whether - # or not they'll use the submodules. This avoids consistency - # issues in the case where submodules are later enabled, but - # someone forgets to check them out. - cmd = '/usr/bin/git submodule update --init --recursive' - status = __salt__['cmd.retcode'](cmd, runas=deploy_user, - umask=002, - cwd=config['location']) - - # http will likely be used as deploy targets' remote transport. - # submodules don't know how to work properly via http - # remotes unless info/refs and other files are up to date. - # This will call git update-server-info for each of the - # checkouts inside of the .git/modules/<modulename> directory. - cmd = ("""git submodule foreach --recursive """ - """'cd $(sed "s/^gitdir: //" .git) && """ - """git update-server-info'""") - - status = __salt__['cmd.retcode'](cmd, runas=deploy_user, - umask=002, - cwd=config['location']) - - # Install a post-checkout hook to run update-server-info - # for each submodule. This command needs to be run - # every time the repository is changed. - hook_directory = os.path.join( - config['location'], '.git', 'hooks' - ) - post_checkout_path = os.path.join( - hook_directory, 'post-checkout' - ) - post_checkout = open(post_checkout_path, 'w') - post_checkout.write(cmd + "\n") - post_checkout.close() - os.chmod(post_checkout_path, 775) - - # we should run this on post-commit too, so just symlink - # post-commit to post-checkout - post_commit_path = os.path.join(hook_directory, 'post-commit') - os.symlink(post_checkout_path, post_commit_path) - # chown the hooks to the deploy_user - deploy_uid = pwd.getpwnam(deploy_user).pw_uid - os.chown(post_checkout_path, deploy_uid, -1) - os.lchown(post_commit_path, deploy_uid, -1) - else: - cmd = '/usr/bin/git init %s' % (config['location']) - status = __salt__['cmd.retcode'](cmd, runas=deploy_user, - umask=002) - if status != 0: - ret_status = 1 - continue - # git clone does ignore umask and does explicit mkdir with 755 - __salt__['file.set_mode'](config['location'], 2775) - - # Set the repo name in the repo's config - cmd = 'git config deploy.repo-name %s' % repo - status = __salt__['cmd.retcode'](cmd, cwd=config['location'], - runas=deploy_user, umask=002) - if status != 0: - ret_status = 1 - continue - # Ensure checkout-submodules is also configured for trigger - if config['checkout_submodules']: - cmd = 'git config deploy.checkout-submodules true' - else: - cmd = 'git config deploy.checkout-submodules false' - status = __salt__['cmd.retcode'](cmd, cwd=config['location'], - runas=deploy_user, umask=002) - if status != 0: - ret_status = 1 - continue - - # Override deploy_group with repo specific value set - # in deployment_config pillar - if config['deployment_repo_group']: - deploy_group = config['deployment_repo_group'] - - if deploy_group is not None: - cmd = 'chown -R %s:%s %s' % (deploy_user, - deploy_group, - config['location']) - status = __salt__['cmd.retcode'](cmd, - cwd=config['location']) - if status != 0: - ret_status = 1 - return ret_status - - -def sync_all(): - ''' - Sync all repositories for this minion. If a repo doesn't exist on target, - clone it as well. This function will ensure all repositories for the - minion are at the current tag as defined by the master and is - be safe to call at any point. - - CLI Example (from the master): - - salt -G 'deployment_target:test' deploy.sync_all - - CLI Example (from a minion): - - salt-call deploy.sync_all - - :rtype: hash - ''' - repo_config = __pillar__.get('repo_config') - deployment_target = __grains__.get('deployment_target') - status = 0 - stats = {} - - for repo in repo_config: - # Ensure the minion is a deployment target for this repo - if repo not in deployment_target: - continue - if repo not in stats: - stats[repo] = {} - stats[repo]["deploy.fetch"] = __salt__['deploy.fetch'](repo) - stats[repo]["deploy.checkout"] = __salt__['deploy.checkout'](repo) - - return {'status': status, 'stats': stats} - - -def _update_gitmodules(config, location, shadow=False): - """ - Finds all .gitmodules in a repository, changes all submodules within them - to point to the correct submodule on the deployment server, then runs - a submodule sync. This function is in support of recursive submodules. - - In the case we need to update a shadow reference repo, the .gitmodules - files will have their submodules point to the reference clone. - - :param config: The config hash for the repo (as pulled from get_config). - :type config: hash - :param location: The location on the filesystem to find the .gitmodules - files. - :type location: str - :param shadow: Defines whether or not this is a shadow reference repo. - :type shadow: bool - :rtype: int - """ - gitmodules_list = __salt__['file.find'](location, name='.gitmodules') - for gitmodules in gitmodules_list: - gitmodules_dir = os.path.dirname(gitmodules) - # Check to see if this is even a repo with submodules. Some repos - # have git repositories checked into the repository and kept the - # git configuration files when doing so. This will cause our submodule - # calls to fail. - cmd = '/usr/bin/git submodule status --quiet' - status = __salt__['cmd.retcode'](cmd, gitmodules_dir) - if status != 0: - continue - # Ensure we're working with an unmodified .gitmodules file - cmd = '/usr/bin/git checkout .gitmodules' - status = __salt__['cmd.retcode'](cmd, gitmodules_dir) - if status != 0: - return status - # Get a list of the submodules - submodules = [] - f = open(gitmodules, 'r') - for line in f.readlines(): - keyval = line.split(' = ') - if keyval[0].strip() == "path": - submodules.append(keyval[1].strip()) - f.close() - if shadow: - # Transform .gitmodules based on reference. Point the submodules - # to the local git repository this repo references. - reference_dir = gitmodules_dir.replace(location, - config['location']) - f = open(gitmodules, 'w') - for submodule in submodules: - f.write('[submodule "{0}"]\n'.format(submodule)) - f.write('\tpath = {0}\n'.format(submodule)) - f.write('\turl = {0}/{1}\n'.format(reference_dir, submodule)) - f.close() - else: - # Transform .gitmodules file based on url. Point the submodules - # to the appropriate place on the deployment server. We can base - # this on a subpath of the repository since the deployment server - # isn't a bare clone. - cmd = '/usr/bin/git config remote.origin.url' - remote = __salt__['cmd.run'](cmd, gitmodules_dir) - if not remote: - return 1 - f = open(gitmodules, 'w') - for submodule in submodules: - submodule_path = 'modules/{0}'.format(submodule) - f.write('[submodule "{0}"]\n'.format(submodule)) - f.write('\tpath = {0}\n'.format(submodule)) - f.write('\turl = {0}/{1}\n'.format(remote, submodule_path)) - f.close() - # Have git update its submodule configuration from the .gitmodules - # file. - cmd = '/usr/bin/git submodule sync' - status = __salt__['cmd.retcode'](cmd, gitmodules_dir) - if status != 0: - return status - return 0 - - -def _gitfat_installed(): - return salt.utils.which('git-fat') - - -def _init_gitfat(location): - ''' - Runs git fat init at this location. - - :param location: The location on the filesystem to run git fat init - :type location: str - ''' - # if it isn't then initialize it now - cmd = '/usr/bin/git fat init' - return __salt__['cmd.retcode'](cmd, location) - - -# TODO: git fat gc? -def _update_gitfat(location): - ''' - Runs git-fat pull at this location. - If git fat has not been initialized for the - repository at this location, _init_gitfat - will be called first. - - :param location: The location on the filesystem to run git fat pull - :type location: str - :rtype int - ''' - - # Make sure git fat is installed. - if not _gitfat_installed(): - return 40 - - # Make sure git fat is initialized. - cmd = '/usr/bin/git config --get filter.fat.smudge' - if __salt__['cmd.run'](cmd, location) != 'git-fat filter-smudge': - status = _init_gitfat(location) - if status != 0: - return status - - # Run git fat pull. - cmd = '/usr/bin/git fat pull' - return __salt__['cmd.retcode'](cmd, location) - - -def _clone(config, location, tag, shadow=False): - """ - Perform a clone of a repo at a specified location, and - do a fetch and checkout of the repo to ensure it's at the - current deployment tag. - - :param config: Config hash as fetched from get_config - :type config: hash - :param location: The location on the filesystem to clone this repo. - :type location: str - :param tag: The tag to ensure this clone is checked out to. - :type tag: str - :param shadow: Whether or not this repo is a shadow reference. - :type shadow: bool - :rtype: int - """ - if shadow: - cmd = '/usr/bin/git clone --reference {0} {1}/.git {2}' - cmd = cmd.format(config['location'], config['url'], location) - else: - cmd = '/usr/bin/git clone {0}/.git {1}'.format(config['url'], location) - status = __salt__['cmd.retcode'](cmd) - if status != 0: - return status - status = _fetch_location(config, location, shadow=shadow) - if status != 0: - return status - status = _checkout_location(config, location, tag, - reset=True, shadow=shadow) - if status != 0: - return status - return 0 - - -def fetch(repo): - ''' - Call a fetch for the specified repo - - CLI Example (from the master): - - salt -G 'deployment_target:test' deploy.fetch 'test/testrepo' - - CLI Example (from the minion): - - salt-call deploy.fetch 'test/testrepo' - - :param repo: The repo on which to perform the fetch. - :type repo: str - :rtype: hash - ''' - config = get_config(repo) - - # Call fetch on all repositories we depend on and add it to the stats for - # reporting back. Deprecated. - depstats = [] - for dependency in config['dependencies']: - depstats.append(__salt__['deploy.fetch'](dependency)) - - # Notify the deployment system we started. - _check_in('deploy.fetch', repo) - - # We need to fetch the tag in case we need to clone and also to ensure the - # fetch has the tag as defined in the deployment. - tag = _get_tag(config) - if not tag: - return {'status': 10, 'repo': repo, 'dependencies': depstats} - - # Clone the repo if it doesn't exist yet otherwise just fetch. Note that - # clone will also properly checkout the repo to the necessary tag, so this - # won't put the repo into an inconsistent state. - if not __salt__['file.directory_exists'](config['location'] + '/.git'): - status = _clone(config, config['location'], tag) - if status != 0: - return {'status': status, 'repo': repo, 'dependencies': depstats} - else: - status = _fetch_location(config, config['location']) - if status != 0: - return {'status': status, 'repo': repo, 'dependencies': depstats} - - # Check to see if the deployment tag has been fetched. - cmd = '/usr/bin/git show-ref refs/tags/{0}'.format(tag) - status = __salt__['cmd.retcode'](cmd, cwd=config['location']) - if status != 0: - return {'status': status, 'repo': repo, 'dependencies': depstats} - - # Do the same steps as above for the shadow reference, but in the case of - # a normal fetch also do a checkout so that fetch_module_calls can use - # a full checkout. - if config['shadow_reference']: - shadow_gitdir = config['shadow_location'] + '/.git' - if not __salt__['file.directory_exists'](shadow_gitdir): - status = _clone(config, config['shadow_location'], tag, - shadow=True) - if status != 0: - return {'status': status, 'repo': repo, - 'dependencies': depstats} - else: - status = _fetch_location(config, config['shadow_location'], - shadow=True) - if status != 0: - return {'status': status, 'repo': repo, - 'dependencies': depstats} - status = _checkout_location(config, config['shadow_location'], tag, - reset=False, shadow=True) - if status != 0: - return {'status': status, 'repo': repo, - 'dependencies': depstats} - - # Call a set of salt modules, but map the args beforehand. - # TODO (ryan-lane): Currently if the module calls fail no error is - # returned and this will silently continue on fail. - # We should modify the config hash to allow for failure - # options. - for call, args in config['fetch_module_calls'].items(): - mapped_args = _map_args(repo, args) - __salt__[call](*mapped_args) - return {'status': status, 'repo': repo, - 'dependencies': depstats, 'tag': tag} - - -def _fetch_location(config, location, shadow=False): - """ - Fetch a repo at a specified location. Optionally define this repo as a - shadow repo. - - :param config: Config hash as fetched from get_config. - :type config: hash - :param location: The location on the filesystem to run the fetch. - :type location: str - :rtype: int - """ - cmd = '/usr/bin/git fetch' - status = __salt__['cmd.retcode'](cmd, location) - if status != 0: - return status - # The deployment tags may not be linked to any branch, so it's safest - # to fetch them explicitly. - cmd = '/usr/bin/git fetch --tags' - status = __salt__['cmd.retcode'](cmd, location) - if status != 0: - return status - - if config['checkout_submodules']: - ret = _update_gitmodules(config, location, shadow) - if ret != 0: - return ret - - # fetch all submodules and tags for submodules - cmd = '/usr/bin/git submodule foreach --recursive git fetch' - status = __salt__['cmd.retcode'](cmd, location) - if status != 0: - return status - # The deployment tags will not be linked to any branch for submodules, - # so it's required to fetch them explicitly. - cmd = '/usr/bin/git submodule foreach --recursive git fetch --tags' - status = __salt__['cmd.retcode'](cmd, location) - if status != 0: - return status - return 0 - - -def _get_tag(config): - """ - Fetch the current deploy file from the repo on the deployment server and - return the current tag associated with it. - - :param config: Config hash as fetched from get_config. - :type config: hash - :rtype: str - """ - # Fetch the .deploy file from the server and get the current tag - deployfile = config['url'] + '/.git/deploy/deploy' - try: - f = urllib.urlopen(deployfile) - deployinfo = f.read() - except IOError: - return None - try: - deployinfo = json.loads(deployinfo) - tag = deployinfo['tag'] - except (KeyError, ValueError): - return None - # tags are user-input and are used in shell commands, ensure they are - # only passing alphanumeric, dashes, or /. - if re.search(r'[^a-zA-Z0-9_\-/]', tag): - return None - return tag - - -def checkout(repo, reset=False): - ''' - Checkout the current deployment tag. Assumes a fetch has been run. - - CLI Example (on master): - - salt -G 'deployment_target:test' deploy.checkout 'test/testrepo' - - CLI Example (on minion): - - salt deploy.checkout 'test/testrepo' - - :param repo: The repo name to check out. - :type repo: str - :param reset: Whether or not to do a checkout and hard reset based - on if the repo is already at the defined tag on the - deployment server. - :type reset: bool - :rtype: hash - ''' - config = get_config(repo) - depstats = [] - status = -1 - - # Notify the deployment system we started - _check_in('deploy.checkout', repo) - - tag = _get_tag(config) - if not tag: - return {'status': status, 'repo': repo, 'dependencies': depstats} - - status = _checkout_location(config, config['location'], tag, reset) - - if status != 0: - return {'status': status, 'repo': repo, 'tag': tag, - 'dependencies': depstats} - - # Call a set of salt modules, but map the args beforehand. - # TODO (ryan-lane): Currently if the module calls fail no error is - # returned and this will silently continue on fail. - # We should modify the config hash to allow for failure - # options. - for call, args in config['checkout_module_calls'].items(): - mapped_args = _map_args(repo, args) - __salt__[call](*mapped_args) - return {'status': status, 'repo': repo, 'tag': tag, - 'dependencies': depstats} - - -def _checkout_location(config, location, tag, reset=False, shadow=False): - """ - Checkout a repo at the specified location to the specified tag. If reset - is true checkout the repo even if it is already at the tag defined on the - deployment server. Optionally specify if this is a shadow reference repo. - - :param config: Config hash as fetched from get_config. - :type config: hash - :param location: The location on the filesystem to run this checkout. - :type location: str - :param tag: The tag to checkout this location to. - :type tag: str - :param reset: Whether or not to checkout this repo if it is already at the - tag specified by the deployment server. - :type reset: bool - :param shadow: Whether or not this is a shadow reference repo. - :type shadow: bool - :rtype: int - """ - # Call checkout on all repositories we depend on and add it to the stats - # for reporting back. Deprecated. - depstats = [] - for dependency in config['dependencies']: - depstats.append(__salt__['deploy.checkout'](dependency, reset)) - - if reset: - # User requested we hard reset the repo to the tag - cmd = '/usr/bin/git reset --hard tags/%s' % (tag) - ret = __salt__['cmd.retcode'](cmd, location) - if ret != 0: - return 20 - else: - # Find the current tag. If it matches the requested deployment tag - # then no further work is needed, just return. - cmd = '/usr/bin/git describe --always --tag' - current_tag = __salt__['cmd.run'](cmd, location) - current_tag = current_tag.strip() - if current_tag == tag: - return 0 - - # Checkout to the tag requested by the deployment. - cmd = '/usr/bin/git checkout --force --quiet tags/%s' % (tag) - ret = __salt__['cmd.retcode'](cmd, location) - if ret != 0: - return 30 - - if config['checkout_submodules']: - ret = _update_gitmodules(config, location, shadow) - if ret != 0: - return ret - - # Update the submodules to match this tag - cmd = '/usr/bin/git submodule update --recursive --init' - ret = __salt__['cmd.retcode'](cmd, location) - if ret != 0: - return 50 - - # Trigger git-fat pull if gitfat_enabled - if config['gitfat_enabled']: - ret = _update_gitfat(location) - if ret != 0: - return ret - - return 0 - - -def restart(repo): - ''' - Restart the service associated with this repo. - - CLI Example (on the master): - - salt -G 'deployment_target:test' deploy.restart 'test/testrepo' - - CLI Example (on the minion): - - salt-call deploy.restart 'test/testrepo' - - :param repo: The repo name used to find the service to restart. - :type repo: str - :rtype: hash - ''' - config = get_config(repo) - _check_in('deploy.restart', repo) - - # Call restart on all repositories we depend on and add it to the stats - # for reporting back. Deprecated. - depstats = [] - for dependency in config['dependencies']: - depstats.append(__salt__['deploy.restart'](dependency)) - - # Get the service associated with this repo and have salt call a restart. - if config['service_name']: - status = __salt__['service.restart'](config['service_name']) - return {'status': status, 'repo': repo, 'dependencies': depstats} - else: - return {} - - -def fixurl(): - """ - Allows to recursively fix all the remotes in git repositories on a target. - """ - repo_config = __pillar__.get('repo_config') - deployment_target = __grains__.get('deployment_target') - for repo in repo_config: - if repo not in deployment_target: - continue - conf = get_config(repo) - # If it has not been checked out, there is no reason to fix the url - if not __salt__['file.directory_exists'](conf['location']): - continue - - cmd = '/usr/bin/git remote set-url origin {0}/.git'.format( - conf['url']) - retval = __salt__['cmd.retcode'](cmd, cwd=conf['location']) - if retval != 0: - return retval - return 0 - - -if __name__ == '__main__': - # Make flake8 happy by defining globals - __pillar__ = dict() - __grains__ = dict() - __salt__ = dict() diff --git a/modules/deployment/files/returners/deploy_redis.py b/modules/deployment/files/returners/deploy_redis.py deleted file mode 100644 index 780b848..0000000 --- a/modules/deployment/files/returners/deploy_redis.py +++ /dev/null @@ -1,92 +0,0 @@ -''' -Return deployment data to a redis server - -To enable this returner the minion will need the python client for redis -installed and the following values configured in the pillar config, these -are the defaults: - - deploy_redis.db: '0' - deploy_redis.host: 'salt' - deploy_redis.port: 6379 -''' - -import redis -import time -import logging - -log = logging.getLogger(__name__) - - -def __virtual__(): - return 'deploy_redis' - - -def _get_serv(): - ''' - Return a redis server object - ''' - deployment_config = __pillar__.get('deployment_config') - deploy_redis = deployment_config['redis'] - socket_connect_timeout = deploy_redis.get('socket_connect_timeout') - if socket_connect_timeout is not None: - socket_connect_timeout = int(socket_connect_timeout) - serv = redis.Redis(host=deploy_redis['host'], - port=int(deploy_redis['port']), - db=int(deploy_redis['db']), - socket_timeout=socket_connect_timeout) - return serv - - -def returner(ret): - ''' - Return data to a redis data store - ''' - function = ret['fun'] - log.debug('Entering deploy_redis returner') - log.debug('function: {0}'.format(function)) - if not function.startswith('deploy.'): - return False - ret_data = ret['return'] - log.debug('ret_data: {0}'.format(ret_data)) - minion = ret['id'] - timestamp = time.time() - serv = _get_serv() - if function == "deploy.sync_all": - for repo, functions in ret_data["stats"].items(): - for func, data in functions.items(): - _record_function(serv, func, timestamp, minion, data) - else: - _record_function(serv, function, timestamp, minion, ret_data) - - -def _record_function(serv, function, timestamp, minion, ret_data): - dependencies = ret_data['dependencies'] - # Record data for all dependent repositories - for dep_data in dependencies: - _record(serv, function, timestamp, minion, dep_data) - # Record data for this repo - _record(serv, function, timestamp, minion, ret_data) - - -def _record(serv, function, timestamp, minion, ret_data): - repo = ret_data['repo'] - minion_key = 'deploy:{0}:minions:{1}'.format(repo, minion) - if function == "deploy.fetch": - if ret_data['status'] == 0: - serv.hset(minion_key, 'fetch_tag', ret_data['tag']) - serv.hset(minion_key, 'fetch_status', ret_data['status']) - serv.hset(minion_key, 'fetch_timestamp', timestamp) - elif function == "deploy.checkout": - if ret_data['status'] == 0: - serv.hset(minion_key, 'tag', ret_data['tag']) - serv.hset(minion_key, 'checkout_status', ret_data['status']) - serv.hset(minion_key, 'checkout_timestamp', timestamp) - elif function == "deploy.restart": - if 'status' in ret_data: - serv.hset(minion_key, 'restart_status', ret_data['status']) - serv.hset(minion_key, 'restart_timestamp', timestamp) - - -if __name__ == '__main__': - # Make flake8 happy by defining globals - __pillar__ = dict() diff --git a/modules/deployment/files/runners/deploy.py b/modules/deployment/files/runners/deploy.py deleted file mode 100755 index 84ed429..0000000 --- a/modules/deployment/files/runners/deploy.py +++ /dev/null @@ -1,66 +0,0 @@ -''' -Authn wrapper for deployment peer calls -''' - -import salt.key -import salt.client - - -def fetch(repo): - ''' - Fetch from a master, for the specified repo - ''' - deployment_target = "deployment_target:" + repo - deployment_server = "deployment_server:*" - targets = "G@{0} and not G@{1}".format(deployment_target, - deployment_server) - client = salt.client.LocalClient(__opts__['conf_file']) - cmd = 'deploy.fetch' - # comma in the tuple is a workaround for a bug in salt - arg = (repo,) - client.cmd(targets, cmd, expr_form='compound', arg=arg, timeout=1, - ret='deploy_redis') - print "Fetch completed" - - -def checkout(repo, reset=False): - ''' - Checkout from a master, for the specified repo - ''' - deployment_target = "deployment_target:" + repo - deployment_server = "deployment_server:*" - targets = "G@{0} and not G@{1}".format(deployment_target, - deployment_server) - client = salt.client.LocalClient(__opts__['conf_file']) - cmd = 'deploy.checkout' - arg = (repo, reset) - client.cmd(targets, cmd, expr_form='compound', arg=arg, timeout=1, - ret='deploy_redis') - print "Checkout completed" - - -def restart(repo, batch='10%', timeout=60): - ''' - Restart the service associated with this repo. If no service is associated - this call will do nothing. - ''' - deployment_target = "deployment_target:" + repo - deployment_server = "deployment_server:*" - targets = "G@{0} and not G@{1}".format(deployment_target, - deployment_server) - client = salt.client.LocalClient(__opts__['conf_file']) - cmd = 'deploy.restart' - # comma in the tuple is a workaround for a bug in salt - arg = (repo,) - ret = [] - for data in client.cmd_batch(targets, cmd, expr_form='compound', - arg=arg, timeout=timeout, - ret='deploy_redis', batch=batch): - ret.append(data) - print "Restart completed" - return ret - - -if __name__ == '__main__': - # Make flake8 happy by defining globals - __opts__ = dict() diff --git a/modules/deployment/files/states/deploy/sync_all.sls b/modules/deployment/files/states/deploy/sync_all.sls deleted file mode 100644 index 075a8b7..0000000 --- a/modules/deployment/files/states/deploy/sync_all.sls +++ /dev/null @@ -1,18 +0,0 @@ -git: - pkg.installed - -python-redis: - pkg.installed - -deploy.sync_all: - ## We need to use cmd.run until module.run supports returners - #module.run: - # - name: deploy.sync_all - # - require: - # - pkg: git - # - pkg: python-redis - cmd.run: - - name: salt-call --return=deploy_redis deploy.sync_all - - require: - - pkg: git - - pkg: python-redis diff --git a/modules/deployment/files/states/top.sls b/modules/deployment/files/states/top.sls deleted file mode 100644 index 0fec40f..0000000 --- a/modules/deployment/files/states/top.sls +++ /dev/null @@ -1,7 +0,0 @@ -base: - 'deployment_server:true': - - match: grain - - deploy.sync_all - 'deployment_target:*': - - match: grain - - deploy.sync_all diff --git a/modules/deployment/manifests/salt_master.pp b/modules/deployment/manifests/salt_master.pp deleted file mode 100644 index 1f6ba3f..0000000 --- a/modules/deployment/manifests/salt_master.pp +++ /dev/null @@ -1,130 +0,0 @@ -class deployment::salt_master( - $repo_config, - $deployment_config, - $state_dir='/srv/salt', - $runner_dir='/srv/runners', - $pillar_dir='/srv/pillars', - $module_dir='/srv/salt/_modules', - $returner_dir='/srv/salt/_returners' -) { - - file { "${state_dir}/deploy": - ensure => directory, - mode => '0555', - owner => 'root', - group => 'root', - require => [File[$state_dir]], - } - - file { "${state_dir}/top.sls": - source => 'puppet:///modules/deployment/states/top.sls', - mode => '0444', - owner => 'root', - group => 'root', - require => [File["${state_dir}/deploy"]], - } - - file { "${state_dir}/deploy/sync_all.sls": - source => 'puppet:///modules/deployment/states/deploy/sync_all.sls', - mode => '0444', - owner => 'root', - group => 'root', - require => [File["${state_dir}/deploy"]], - } - - file { "${runner_dir}/deploy.py": - source => 'puppet:///modules/deployment/runners/deploy.py', - mode => '0555', - owner => 'root', - group => 'root', - require => File[$runner_dir], - } - - file { "${pillar_dir}/deployment": - ensure => directory, - mode => '0555', - owner => 'root', - group => 'root', - require => [File[$pillar_dir]], - } - - file { "${pillar_dir}/deployment/repo_config.sls": - # lint:ignore:arrow_alignment - content => ordered_json({'repo_config' => $repo_config }), - # lint:endignore - mode => '0444', - owner => 'root', - group => 'root', - require => [File["${pillar_dir}/deployment"]], - } - - file { "${pillar_dir}/deployment/deployment_config.sls": - # lint:ignore:arrow_alignment - content => ordered_json({'deployment_config' => $deployment_config}), - # lint:endignore - mode => '0444', - owner => 'root', - group => 'root', - require => [File["${pillar_dir}/deployment"]], - } - - file { "${pillar_dir}/top.sls": - content => template('deployment/pillars/top.sls.erb'), - mode => '0444', - owner => 'root', - group => 'root', - require => [File[$pillar_dir]], - } - - file { "${module_dir}/deploy.py": - source => 'puppet:///modules/deployment/modules/deploy.py', - mode => '0555', - owner => 'root', - group => 'root', - require => [File[$module_dir]], - } - - file { "${returner_dir}/deploy_redis.py": - source => 'puppet:///modules/deployment/returners/deploy_redis.py', - mode => '0555', - owner => 'root', - group => 'root', - require => [File[$returner_dir]], - } - - # deprecated in T97509 - file { "${module_dir}/mwprof.py": - ensure => absent, - } - - # If pillars or modules change, we need to sync them with the minions - exec { 'refresh_deployment_pillars': - command => "/usr/bin/salt -C 'G@deployment_server:true or G@deployment_target:*' saltutil.refresh_pillar", - subscribe => [File["${pillar_dir}/deployment/deployment_config.sls"], - File["${pillar_dir}/deployment/repo_config.sls"], - File[$pillar_dir]], - refreshonly => true, - require => [Package['salt-master']], - } - - exec { 'deployment_server_init': - command => "/usr/bin/salt -G 'deployment_server:true' deploy.deployment_server_init", - subscribe => [Exec['refresh_deployment_pillars']], - refreshonly => true, - require => [File["${module_dir}/deploy.py"]], - } - - exec { 'refresh_deployment_modules': - command => "/usr/bin/salt -C 'G@deployment_server:true or G@deployment_target:*' saltutil.sync_modules", - subscribe => File["${module_dir}/deploy.py"], - refreshonly => true, - require => [Package['salt-master']], - } - - exec { 'refresh_deployment_returners': - command => "/usr/bin/salt -C 'G@deployment_server:true or G@deployment_target:*' saltutil.sync_returners", - subscribe => [File["${returner_dir}/deploy_redis.py"]], - refreshonly => true, - require => [Package['salt-master']], - } -} diff --git a/modules/deployment/templates/pillars/top.sls.erb b/modules/deployment/templates/pillars/top.sls.erb deleted file mode 100644 index 1bfd986..0000000 --- a/modules/deployment/templates/pillars/top.sls.erb +++ /dev/null @@ -1,9 +0,0 @@ -base: - 'deployment_server:true': - - match: grain - - deployment.repo_config - - deployment.deployment_config - 'deployment_target:*': - - match: grain - - deployment.repo_config - - deployment.deployment_config diff --git a/modules/role/manifests/deployment/salt_masters.pp b/modules/role/manifests/deployment/salt_masters.pp deleted file mode 100644 index eeaa0b4..0000000 --- a/modules/role/manifests/deployment/salt_masters.pp +++ /dev/null @@ -1,8 +0,0 @@ -# === Class role::deployment::salt_masters -# Installs deployment-related data to the salt master -# -# filtertags: labs-project-servermon -class role::deployment::salt_masters( - $deployment_server = hiera('deployment_server', 'tin.eqiad.wmnet'), -) { -} -- To view, visit https://gerrit.wikimedia.org/r/379189 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ide7b78e46617338ff9e044a6939519afd40d404e Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Muehlenhoff <mmuhlenh...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
