Hello Madhuvishy, Chasemp, Yuvipanda,
I'd like you to do a code review. Please visit
https://gerrit.wikimedia.org/r/336157
to review the following change.
Change subject: labstore: Remove create-dbusers
......................................................................
labstore: Remove create-dbusers
create-dbusers was replaced by maintain-dbusers in
c5f0338767a4394b3d82de3283455379f62eee21. This change removes the
remnant and now unused file and a reference to it that only made sense
during the migration.
Change-Id: I1cf97904cc20582d256ccb3c3e7e74c720a3a80a
---
D modules/labstore/files/create-dbusers
M modules/role/files/labs/db/maintain-dbusers.py
2 files changed, 1 insertion(+), 334 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/57/336157/1
diff --git a/modules/labstore/files/create-dbusers
b/modules/labstore/files/create-dbusers
deleted file mode 100755
index edea94a..0000000
--- a/modules/labstore/files/create-dbusers
+++ /dev/null
@@ -1,332 +0,0 @@
-#!/usr/bin/python3
-"""
-This script does the following:
-
- - Check if users / service groups in opted in projects have
- a replica.my.cnf file with mysql credentials
- - If they do not exist, create a mysql user, give them
- appropriate grants and write the replica.my.cnf file
- - If there is no replica.my.cnf but grants already exist, do
- nothing. This needs to be fixed with appropriate solution
- later on - either by recreating the replica.my.cnf or...
- something else
- - perms for grant files (how the user gets the creds + password)
- are 0400 and chattr +i making them user readonly and immutable.
-"""
-import logging
-import argparse
-import ldap3
-import pymysql
-import yaml
-import os
-import string
-import random
-import configparser
-import io
-import time
-import subprocess
-
-
-class User:
-
- def __init__(self, project, name, uid, kind):
- self.project = project
- self.name = name
- self.uid = int(uid)
-
- if kind not in ['user', 'servicegroup']:
- raise("{} not allowed".format(kind))
-
- self.kind = kind
-
- self.project_root = '/exp/project'
-
- @property
- def db_username(self):
- """
- The db username to use for this user.
-
- Guaranteed to be of the form (s|u)\d+
- """
- prefix = 'u' if self.kind == 'user' else 's'
- return prefix + str(self.uid)
-
- @property
- def homedir(self):
- prefix = os.path.join(self.project_root, self.project)
-
- if self.kind == 'user':
- trail = os.path.join(prefix,
- 'home',
- self.name,
- )
- else:
- trail = os.path.join(prefix,
- 'project',
- self.name[len(self.project) + 1:],
- )
-
- return trail
-
- def __repr__(self):
- return "%s(name=%s, uid=%s)" % (
- self.kind, self.name, self.uid)
-
- @classmethod
- def from_ldap_servicegroups(cls, conn, project):
-
- logging.debug('Collecting servicegroups from ldap for
{}'.format(project))
- conn.search(
- 'ou=people,ou=servicegroups,dc=wikimedia,dc=org',
- '(cn=%s.*)' % project,
- ldap3.SEARCH_SCOPE_WHOLE_SUBTREE,
- attributes=['uidNumber', 'cn'],
- time_limit=5
- )
-
- users = []
- for resp in conn.response:
- attrs = resp['attributes']
- users.append(cls(project, attrs['cn'][0], attrs['uidNumber'][0],
'servicegroup'))
-
- logging.debug('Found {} ldap servicegroups'.format(len(users)))
- logging.debug('First 10 members of ldap servicegroups:
{}'.format(users[:10]))
- return users
-
- @classmethod
- def from_ldap_users(cls, conn, project):
-
- logging.debug('Collecting ldap users for {}'.format(project))
- conn.search(
- 'ou=groups,dc=wikimedia,dc=org',
- '(cn=project-%s)' % project,
- ldap3.SEARCH_SCOPE_WHOLE_SUBTREE,
- attributes=['member']
- )
-
- users = []
- members = conn.response[0]['attributes']['member']
-
- logging.debug('Found {} ldap users'.format(len(members)))
- logging.debug('First 10 members of ldap users: {}
...'.format(members[:10]))
-
- # example member: uid=foo,ou=people,dc=wikimedia,dc=org
- for member in members:
-
- # strip off all but 'uid=foo'
- search_string = member.replace(',ou=people,dc=wikimedia,dc=org',
'')
-
- conn.search(
- 'ou=people,dc=wikimedia,dc=org',
- '(%s)' % search_string,
- ldap3.SEARCH_SCOPE_WHOLE_SUBTREE,
- attributes=['uid', 'uidNumber']
- )
-
- # Example of ldap query response:
- # [
- # {'type': 'searchResEntry',
- # 'attributes': {
- # 'uid': ['foo'],
- # 'uidNumber': ['1111']},
- # 'raw_attributes': {
- # 'uid': [b'foo'],
- # 'uidNumber': [b'1111']
- # },
- # 'dn': 'uid=foo,ou=people,dc=wikimedia,dc=org'}
- # ]
-
- if len(conn.response) == 0:
- logging.error(
- 'No entry found for user {user} in project
{project}'.format(
- user=member,
- project=project,
- )
- )
- continue
-
- attrs = conn.response[0]['attributes']
- users.append(cls(
- project,
- attrs['uid'][0],
- attrs['uidNumber'][0],
- 'user'
- ))
-
- logging.debug('Found {} ldap users with uidNumer'.format(len(users)))
- return users
-
- def write_user_file(self, path, content):
- logging.debug('writing path %s' % (path,))
- f = os.open(path, os.O_CREAT | os.O_WRONLY | os.O_NOFOLLOW)
- try:
- os.write(f, content.encode('utf-8'))
- # uid == gid
- os.fchown(f, self.uid, self.uid)
- os.fchmod(f, 0o400)
-
- # Prevent removal or modification of the credentials file
- subprocess.check_output(['/usr/bin/chattr',
- '+i',
- path])
- except:
- os.remove(path)
- raise
- finally:
- os.close(f)
-
-
-class CredentialCreator:
- PASSWORD_LENGTH = 16
- PASSWORD_CHARS = string.ascii_letters + string.digits
-
- GRANT_SQL_TEMPLATE = """
- CREATE USER '{user_name}'@'%' IDENTIFIED BY '{user_pass}';
- GRANT SELECT, SHOW VIEW ON `%\_p`.* TO '{user_name}'@'%';
- GRANT ALL PRIVILEGES ON `{user_name}\_\_%`.* TO '{user_name}'@'%';"""
-
- def __init__(self, hosts, username, password):
- self.conns = [
- pymysql.connect(host, username, password)
- for host in hosts
- ]
-
- @staticmethod
- def _generate_pass():
- sysrandom = random.SystemRandom() # Uses /dev/urandom
- return ''.join(sysrandom.sample(
- CredentialCreator.PASSWORD_CHARS,
- CredentialCreator.PASSWORD_LENGTH))
-
- def write_credentials_file(self, path, user):
-
- password = self._generate_pass()
- replica_config = configparser.ConfigParser()
-
- replica_config['client'] = {
- 'user': user.db_username,
- 'password': password
- }
-
- self.create_user(user, password)
- # Because ConfigParser can only write to a file
- # and not just return the value as a string directly
- replica_buffer = io.StringIO()
- replica_config.write(replica_buffer)
-
- logging.info("creating replica.my.cnf for %s as %s", user.name,
user.db_username)
- user.write_user_file(path, replica_buffer.getvalue())
- logging.debug('write: {} {}\n'.format(path, replica_buffer.getvalue()))
-
- def check_user_exists(self, user):
- exists = True
- for conn in self.conns:
- conn.ping(True)
- cur = conn.cursor()
- try:
- cur.execute('SELECT * FROM mysql.user WHERE User = %s',
user.db_username)
- result = cur.fetchone()
- finally:
- cur.close()
- exists = exists and (result is not None)
- return exists
-
- def create_user(self, user, password):
- for conn in self.conns:
- conn.ping(True)
- cur = conn.cursor()
- try:
- # is ok, because password is guaranteed to never
- # contain a quote (only alphanumeric) and username
- # is guaranteed to be (u|s)\d+.
- sql = CredentialCreator.GRANT_SQL_TEMPLATE.format(
- user_name=user.db_username,
- user_pass=password
- )
-
- logging.debug('sql: {}'.format(str(sql)))
- cur.execute(sql)
- logging.info('Created user %s as %s in %s', user.name,
user.db_username, conn.host)
-
- finally:
- cur.close()
-
-
-if __name__ == '__main__':
-
- argparser = argparse.ArgumentParser()
-
- argparser.add_argument('--config',
- default='/etc/create-dbusers.yaml',
- help='Path to YAML config file')
-
- argparser.add_argument('--debug',
- help='Turn on debug logging',
- action='store_true')
-
- argparser.add_argument('--project',
- help='Project name to create db users for',
- default='tools')
-
- argparser.add_argument('--interval',
- help='Seconds between between runs',
- type=int,
- default=0)
-
- args = argparser.parse_args()
-
- loglvl = logging.DEBUG if args.debug else logging.INFO
- logging.basicConfig(format='%(message)s',
- level=loglvl)
-
- with open(args.config) as f:
- config = yaml.safe_load(f)
-
- labsdb_hosts = [
- k for k in config['labsdbs']['hosts']
- if config['labsdbs']['hosts'][k]['grant-type'] == 'legacy'
- ]
- cgen = CredentialCreator(
- labsdb_hosts,
- config['labsdbs']['username'],
- config['labsdbs']['password']
- )
-
- pid = os.getpid()
- logging.info('starting pid %s create-dbusers run' % (pid,))
- logging.debug(str(args))
-
- while True:
-
- servers = ldap3.ServerPool([
- ldap3.Server(host, connect_timeout=1)
- for host in config['ldap']['hosts']
- ], ldap3.POOLING_STRATEGY_ROUND_ROBIN, active=True, exhaust=True)
-
- with ldap3.Connection(
- servers, read_only=True,
- user=config['ldap']['username'],
- auto_bind=True,
- password=config['ldap']['password']
- ) as conn:
- users = User.from_ldap_users(conn, args.project)
- servicegroups = User.from_ldap_servicegroups(conn,
args.project)
- all_users = users + servicegroups
-
- for user in all_users:
- replica_path = os.path.join(user.homedir, 'replica.my.cnf')
- #logging.debug(replica_path)
- if os.path.exists(user.homedir) and not
os.path.exists(replica_path):
- if not cgen.check_user_exists(user):
- # No replica.my.cnf and no user in db
- # Generate new creds and put them in there!
- cgen.write_credentials_file(replica_path, user)
- else:
- logging.debug('missing replica.my.cnf for user %s despite
grants present in db', user.name)
-
- logging.info('completed pid %s create-dbusers run' % (pid,))
-
- if not args.interval:
- break
-
- time.sleep(args.interval)
diff --git a/modules/role/files/labs/db/maintain-dbusers.py
b/modules/role/files/labs/db/maintain-dbusers.py
index 2a7386e..56accf2 100644
--- a/modules/role/files/labs/db/maintain-dbusers.py
+++ b/modules/role/files/labs/db/maintain-dbusers.py
@@ -9,8 +9,7 @@
mutate the DB in some way. They are also supposed to be idempotent -
if they have nothing to do, they should not do anything.
-Some of the functions are one-time only, allowing migration from
-create-dbusers. These are:
+Some of the functions are one-time only. These are:
## harvest_cnf_files ##
--
To view, visit https://gerrit.wikimedia.org/r/336157
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I1cf97904cc20582d256ccb3c3e7e74c720a3a80a
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Tim Landscheidt <t...@tim-landscheidt.de>
Gerrit-Reviewer: Chasemp <r...@wikimedia.org>
Gerrit-Reviewer: Madhuvishy <mviswanat...@wikimedia.org>
Gerrit-Reviewer: Yuvipanda <yuvipa...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
