Rush has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/369809 )
Change subject: openstack: clientlib refactor ...................................................................... openstack: clientlib refactor Bug: T171494 Change-Id: I491a04889097d286c551d2ca16d719b5efc8a604 --- M hieradata/labs.yaml M modules/labstore/manifests/fileserver/exports.pp M modules/openstack/manifests/keystonechecks.pp A modules/openstack2/files/clientlib/mwopenstackclients.py R modules/openstack2/manifests/clientlib.pp A modules/profile/manifests/openstack/base/clientlib.pp A modules/profile/manifests/openstack/labtest/clientlib.pp A modules/profile/manifests/openstack/labtestn/clientlib.pp A modules/profile/manifests/openstack/main/clientlib.pp M modules/role/manifests/horizon.pp M modules/role/manifests/labs/nfs/secondary.pp M modules/role/manifests/labs/openstack/keystone/server.pp M modules/role/manifests/labs/openstack/nova/controller.pp M modules/role/manifests/labs/openstack/nova/manager.pp M modules/role/manifests/labs/puppetmaster/frontend.pp M modules/role/manifests/labs/shinken.pp M modules/role/manifests/toollabs/clush/master.pp M modules/role/manifests/wmcs/openstack/labtest/control.pp M modules/role/manifests/wmcs/openstack/labtest/web.pp A modules/role/manifests/wmcs/openstack/labtest/wikitech.pp M modules/role/manifests/wmcs/openstack/labtestn/control.pp M modules/role/manifests/wmcs/openstack/labtestn/web.pp M modules/role/manifests/wmcs/openstack/main/control.pp M modules/role/manifests/wmcs/openstack/main/net.pp M modules/role/manifests/wmcs/openstack/main/web.pp M modules/role/manifests/wmcs/openstack/main/wikitech.pp M modules/shinken/manifests/shinkengen.pp 27 files changed, 233 insertions(+), 35 deletions(-) Approvals: Rush: Looks good to me, approved jenkins-bot: Verified diff --git a/hieradata/labs.yaml b/hieradata/labs.yaml index 0d265da..d6564be3 100644 --- a/hieradata/labs.yaml +++ b/hieradata/labs.yaml @@ -17,6 +17,7 @@ recursor: 'labs-recursor0.wikimedia.org' recursor_secondary: 'labs-recursor1.wikimedia.org' +profile::openstack::main::version: 'liberty' profile::openstack::base::region: "%{::site}" profile::openstack::main::nova_controller: 'labcontrol1001.wikimedia.org' profile::openstack::base::observer_user: 'novaobserver' diff --git a/modules/labstore/manifests/fileserver/exports.pp b/modules/labstore/manifests/fileserver/exports.pp index e932db8..aa3869c 100644 --- a/modules/labstore/manifests/fileserver/exports.pp +++ b/modules/labstore/manifests/fileserver/exports.pp @@ -52,7 +52,6 @@ } - include ::openstack::clientlib file { '/usr/local/bin/nfs-exportd': owner => 'root', group => 'root', diff --git a/modules/openstack/manifests/keystonechecks.pp b/modules/openstack/manifests/keystonechecks.pp index 7864031..7b3cef8 100644 --- a/modules/openstack/manifests/keystonechecks.pp +++ b/modules/openstack/manifests/keystonechecks.pp @@ -4,7 +4,6 @@ # # This also checks the functionality of the keystone API generally. class openstack::keystonechecks() { - include ::openstack::clientlib # Script to check all keystone projects for a given user and role file { '/usr/local/bin/check_keystone_roles.py': diff --git a/modules/openstack2/files/clientlib/mwopenstackclients.py b/modules/openstack2/files/clientlib/mwopenstackclients.py new file mode 100644 index 0000000..68d8c54 --- /dev/null +++ b/modules/openstack2/files/clientlib/mwopenstackclients.py @@ -0,0 +1,138 @@ +import os +import yaml + +import glanceclient +from keystoneclient.auth.identity import generic +from keystoneclient import session as keystone_session +from keystoneclient.v3 import client as keystone_client +from novaclient import client as nova_client + + +class Clients(object): + # envfile should be a puppetized environment file like observerenv.sh. + # + # If envfile is not specified, specific creds can be passed in as + # username, password, url, project args. Failing that we fall + # back on the environment. + def __init__(self, + envfile="", + username="", + password="", + url="", + project=""): + self.sessions = {} + self.keystoneclients = {} + self.novaclients = {} + self.glanceclients = {} + + if envfile: + if username or password or url or project: + raise Exception("envfile is incompatible with specific args") + + with open(envfile) as f: + env = yaml.safe_load(f) + self.username = env['OS_USERNAME'] + self.password = env['OS_PASSWORD'] + self.url = env['OS_AUTH_URL'] + self.project = env['OS_TENANT_NAME'] + else: + if username: + self.username = username + else: + self.username = os.environ.get('OS_USERNAME', None) + + if password: + self.password = password + else: + self.password = os.environ.get('OS_PASSWORD', None) + + if url: + self.url = url + else: + self.url = os.environ.get('OS_AUTH_URL', None) + + if project: + self.project = project + else: + self.project = os.environ.get('OS_TENANT_NAME', None) + + if not self.username: + raise Exception("No username (env OS_USERNAME) specified") + if not self.password: + raise Exception("No password (env OS_PASSWORD) specified") + if not self.url: + raise Exception("No url (env OS_AUTH_URL) specified") + if not self.project: + raise Exception("No project (env OS_TENANT_NAME) specified") + + def session(self, project=None): + if not project: + project = self.project + + if project not in self.sessions: + + auth = generic.Password( + auth_url=self.url, + username=self.username, + password=self.password, + user_domain_name='Default', + project_domain_name='Default', + project_name=project) + + self.sessions[project] = keystone_session.Session(auth=auth) + return self.sessions[project] + + def keystoneclient(self, project=None): + if not project: + project = self.project + + if project not in self.keystoneclients: + session = self.session(project) + self.keystoneclients[project] = keystone_client.Client( + session=session, interface='public', connect_retries=5) + return self.keystoneclients[project] + + def novaclient(self, project=None): + if not project: + project = self.project + + if project not in self.novaclients: + session = self.session(project) + self.novaclients[project] = nova_client.Client('2', + session=session, + connect_retries=5) + return self.novaclients[project] + + def glanceclient(self, project=None): + if not project: + project = self.project + + if project not in self.glanceclients: + session = self.session(project) + self.glanceclients[project] = glanceclient.Client('1', + session=session, + connect_retries=5) + return self.glanceclients[project] + + def allprojects(self): + client = self.keystoneclient() + return client.projects.list() + + def allinstances(self, projectid=None): + if projectid: + return self.novaclient(projectid).servers.list() + else: + instances = [] + for project in self.allprojects(): + if project.id == 'admin': + continue + instances.extend(self.novaclient(project.id).servers.list()) + return instances + + def globalimages(self): + client = self.glanceclient() + return [i for i in client.images.list()] + + +# Alias class for back-compat with old consumers +clients = Clients diff --git a/modules/openstack/manifests/clientlib.pp b/modules/openstack2/manifests/clientlib.pp similarity index 75% rename from modules/openstack/manifests/clientlib.pp rename to modules/openstack2/manifests/clientlib.pp index d202019..e29da76 100644 --- a/modules/openstack/manifests/clientlib.pp +++ b/modules/openstack2/manifests/clientlib.pp @@ -1,7 +1,7 @@ # Utilities for querying openstack -class openstack::clientlib { - include ::openstack - require openstack2::cloudrepo +class openstack2::clientlib( + $version, + ) { $packages = [ 'python-novaclient', @@ -14,13 +14,14 @@ # Wrapper python class to easily query openstack clients file { '/usr/lib/python2.7/dist-packages/mwopenstackclients.py': ensure => present, - source => 'puppet:///modules/openstack/mwopenstackclients.py', + source => 'puppet:///modules/openstack2/clientlib/mwopenstackclients.py', mode => '0755', owner => 'root', group => 'root', } - if $::openstack::version != 'liberty' { + # assumption is any version not liberty is newer + if $version != 'liberty' { # Python3 client packages are only available in Mitaka # and later repos @@ -33,7 +34,7 @@ file { '/usr/lib/python3/dist-packages/mwopenstackclients.py': ensure => present, - source => 'puppet:///modules/openstack/mwopenstackclients.py', + source => 'puppet:///modules/openstack2/clientlib/mwopenstackclients.py', mode => '0755', owner => 'root', group => 'root', diff --git a/modules/profile/manifests/openstack/base/clientlib.pp b/modules/profile/manifests/openstack/base/clientlib.pp new file mode 100644 index 0000000..fdfe1c9 --- /dev/null +++ b/modules/profile/manifests/openstack/base/clientlib.pp @@ -0,0 +1,8 @@ +class profile::openstack::base::clientlib( + $version = hiera('profile::openstack::base::version'), + ) { + + class {'openstack2::clientlib': + version => $version + } +} diff --git a/modules/profile/manifests/openstack/labtest/clientlib.pp b/modules/profile/manifests/openstack/labtest/clientlib.pp new file mode 100644 index 0000000..39889e0 --- /dev/null +++ b/modules/profile/manifests/openstack/labtest/clientlib.pp @@ -0,0 +1,9 @@ +class profile::openstack::labtest::clientlib( + $version = hiera('profile::openstack::labtest::version'), + ) { + + require ::profile::openstack::labtest::cloudrepo + class {'profile::openstack::base::clientlib': + version => $version + } +} diff --git a/modules/profile/manifests/openstack/labtestn/clientlib.pp b/modules/profile/manifests/openstack/labtestn/clientlib.pp new file mode 100644 index 0000000..c21be9e --- /dev/null +++ b/modules/profile/manifests/openstack/labtestn/clientlib.pp @@ -0,0 +1,9 @@ +class profile::openstack::labtestn::clientlib( + $version = hiera('profile::openstack::labtestn::version'), + ) { + + require ::profile::openstack::labtestn::cloudrepo + class {'profile::openstack::base::clientlib': + version => $version + } +} diff --git a/modules/profile/manifests/openstack/main/clientlib.pp b/modules/profile/manifests/openstack/main/clientlib.pp new file mode 100644 index 0000000..0b88aa3 --- /dev/null +++ b/modules/profile/manifests/openstack/main/clientlib.pp @@ -0,0 +1,9 @@ +class profile::openstack::main::clientlib( + $version = hiera('profile::openstack::main::version'), + ) { + + require ::profile::openstack::main::cloudrepo + class {'profile::openstack::base::clientlib': + version => $version + } +} diff --git a/modules/role/manifests/horizon.pp b/modules/role/manifests/horizon.pp index 882f55e..f116d74 100644 --- a/modules/role/manifests/horizon.pp +++ b/modules/role/manifests/horizon.pp @@ -15,7 +15,6 @@ srange => '$PRODUCTION_NETWORKS', } - include ::openstack::clientlib class { '::openstack::envscripts': novaconfig => $novaconfig, designateconfig => $designateconfig diff --git a/modules/role/manifests/labs/nfs/secondary.pp b/modules/role/manifests/labs/nfs/secondary.pp index c9c2942..5824ae6 100644 --- a/modules/role/manifests/labs/nfs/secondary.pp +++ b/modules/role/manifests/labs/nfs/secondary.pp @@ -7,6 +7,7 @@ description => 'NFS secondary share cluster', } + require ::profile::openstack::main::clientlib require ::profile::openstack::main::observerenv include labstore::fileserver::exports include labstore::fileserver::secondary diff --git a/modules/role/manifests/labs/openstack/keystone/server.pp b/modules/role/manifests/labs/openstack/keystone/server.pp index 616b312..b8e44fc 100644 --- a/modules/role/manifests/labs/openstack/keystone/server.pp +++ b/modules/role/manifests/labs/openstack/keystone/server.pp @@ -1,7 +1,6 @@ class role::labs::openstack::keystone::server { system::role { $name: } - $nova_controller = hiera('labs_nova_controller') $keystoneconfig = hiera_hash('keystoneconfig', {}) $wikitechstatusconfig = hiera_hash('wikitechstatusconfig', {}) @@ -9,7 +8,4 @@ class { 'openstack::keystone::service': keystoneconfig => $keystoneconfig, } - - # Monitor project membership - include ::openstack::clientlib } diff --git a/modules/role/manifests/labs/openstack/nova/controller.pp b/modules/role/manifests/labs/openstack/nova/controller.pp index f6096c0..9c1d9ee 100644 --- a/modules/role/manifests/labs/openstack/nova/controller.pp +++ b/modules/role/manifests/labs/openstack/nova/controller.pp @@ -8,7 +8,6 @@ include ::openstack::nova::conductor include ::openstack::nova::spiceproxy include ::openstack::nova::scheduler - include ::openstack::clientlib include role::labs::openstack::nova::common $novaconfig = $role::labs::openstack::nova::common::novaconfig $designateconfig = hiera_hash('designateconfig', {}) diff --git a/modules/role/manifests/labs/openstack/nova/manager.pp b/modules/role/manifests/labs/openstack/nova/manager.pp index 9e757da..fbcec2a 100644 --- a/modules/role/manifests/labs/openstack/nova/manager.pp +++ b/modules/role/manifests/labs/openstack/nova/manager.pp @@ -1,13 +1,14 @@ # This is the wikitech UI class role::labs::openstack::nova::manager { + system::role { $name: } + include ::nutcracker::monitoring include ::mediawiki::packages::php5 include ::mediawiki::packages::math include ::mediawiki::packages::tex include ::mediawiki::cgroup include ::scap::scripts - include ::openstack::clientlib include role::labs::openstack::nova::common $novaconfig = $role::labs::openstack::nova::common::novaconfig diff --git a/modules/role/manifests/labs/puppetmaster/frontend.pp b/modules/role/manifests/labs/puppetmaster/frontend.pp index 155c902..f494198 100644 --- a/modules/role/manifests/labs/puppetmaster/frontend.pp +++ b/modules/role/manifests/labs/puppetmaster/frontend.pp @@ -1,6 +1,7 @@ # vim: set tabstop=4 shiftwidth=4 softtabstop=4 expandtab textwidth=80 smarttab class role::labs::puppetmaster::frontend() { + system::role { 'puppetmaster': description => 'Puppetmaster frontend' } @@ -13,6 +14,18 @@ $horizon_host_ip = ipresolve(hiera('labs_horizon_host'), 4) $designate_host_ip = ipresolve(hiera('labs_designate_hostname'), 4) + # XXX: break this up into profiles and + # then make roles for each openstack deployment for hiera + $openstack_version = 'liberty' + class {'openstack2::cloudrepo': + version => $openstack_version + } + + class {'openstack2::clientlib': + version => $openstack_version, + require => Class['openstack2::cloudrepo'], + } + # Only allow puppet access from the instances $allow_from = flatten([$labs_instance_range, $labs_metal, '.wikimedia.org']) @@ -24,7 +37,6 @@ # validatelabsfqdn will look up an instance certname in nova # and make sure it's for an actual instance before signing - include ::openstack::clientlib file { '/usr/local/sbin/validatelabsfqdn.py': ensure => 'present', owner => 'root', diff --git a/modules/role/manifests/labs/shinken.pp b/modules/role/manifests/labs/shinken.pp index 33d552e..30f3bc2 100644 --- a/modules/role/manifests/labs/shinken.pp +++ b/modules/role/manifests/labs/shinken.pp @@ -7,9 +7,13 @@ # Setup an ircbot using ircecho to support echoing notifications # # filtertags: labs-project-shinken + class role::labs::shinken( $ircbot = true, ){ + + require ::profile::openstack::main::clientlib + class { '::shinken': auth_secret => 'This is insecure, should switch to using private repo', } diff --git a/modules/role/manifests/toollabs/clush/master.pp b/modules/role/manifests/toollabs/clush/master.pp index 655146d..7fd784c 100644 --- a/modules/role/manifests/toollabs/clush/master.pp +++ b/modules/role/manifests/toollabs/clush/master.pp @@ -18,6 +18,8 @@ # # filtertags: labs-project-toolsbeta labs-project-tools class role::toollabs::clush::master { + + require ::profile::openstack::main::clientlib include ::toollabs::infrastructure class { '::clush::master': @@ -26,7 +28,6 @@ require_package('python3-yaml') - include ::openstack::clientlib file { '/usr/local/sbin/tools-clush-generator': ensure => file, source => 'puppet:///modules/role/toollabs/clush/tools-clush-generator', diff --git a/modules/role/manifests/wmcs/openstack/labtest/control.pp b/modules/role/manifests/wmcs/openstack/labtest/control.pp index 58ba38a..b227a6a 100644 --- a/modules/role/manifests/wmcs/openstack/labtest/control.pp +++ b/modules/role/manifests/wmcs/openstack/labtest/control.pp @@ -1,5 +1,6 @@ class role::wmcs::openstack::labtest::control { - include profile::openstack::labtest::cloudrepo - include profile::openstack::labtest::observerenv - include profile::openstack::labtest::rabbitmq + include ::profile::openstack::labtest::cloudrepo + include ::profile::openstack::labtest::clientlib + include ::profile::openstack::labtest::observerenv + include ::profile::openstack::labtest::rabbitmq } diff --git a/modules/role/manifests/wmcs/openstack/labtest/web.pp b/modules/role/manifests/wmcs/openstack/labtest/web.pp index afe28cf..4b5deca 100644 --- a/modules/role/manifests/wmcs/openstack/labtest/web.pp +++ b/modules/role/manifests/wmcs/openstack/labtest/web.pp @@ -1,4 +1,5 @@ class role::wmcs::openstack::labtest::web { - include profile::openstack::labtest::observerenv - include profile::openstack::labtest::cloudrepo + include ::profile::openstack::labtest::cloudrepo + include ::profile::openstack::labtest::clientlib + include ::profile::openstack::labtest::observerenv } diff --git a/modules/role/manifests/wmcs/openstack/labtest/wikitech.pp b/modules/role/manifests/wmcs/openstack/labtest/wikitech.pp new file mode 100644 index 0000000..68fbf09 --- /dev/null +++ b/modules/role/manifests/wmcs/openstack/labtest/wikitech.pp @@ -0,0 +1,4 @@ +class role::wmcs::openstack::labtest::wikitech { + include ::profile::openstack::labtest::cloudrepo + include ::profile::openstack::labtest::clientlib +} diff --git a/modules/role/manifests/wmcs/openstack/labtestn/control.pp b/modules/role/manifests/wmcs/openstack/labtestn/control.pp index 82cb4f5..c85dc28 100644 --- a/modules/role/manifests/wmcs/openstack/labtestn/control.pp +++ b/modules/role/manifests/wmcs/openstack/labtestn/control.pp @@ -1,5 +1,6 @@ class role::wmcs::openstack::labtestn::control { - include profile::openstack::labtestn::cloudrepo - include profile::openstack::labtestn::observerenv - include profile::openstack::labtestn::rabbitmq + include ::profile::openstack::labtestn::cloudrepo + include ::profile::openstack::labtestn::clientlib + include ::profile::openstack::labtestn::observerenv + include ::profile::openstack::labtestn::rabbitmq } diff --git a/modules/role/manifests/wmcs/openstack/labtestn/web.pp b/modules/role/manifests/wmcs/openstack/labtestn/web.pp index f20496d..439cc40 100644 --- a/modules/role/manifests/wmcs/openstack/labtestn/web.pp +++ b/modules/role/manifests/wmcs/openstack/labtestn/web.pp @@ -1,4 +1,5 @@ class role::wmcs::openstack::labtestn::web { - include profile::openstack::labtestn::observerenv - include profile::openstack::labtestn::cloudrepo + include ::profile::openstack::labtestn::cloudrepo + include ::profile::openstack::labtestn::clientlib + include ::profile::openstack::labtestn::observerenv } diff --git a/modules/role/manifests/wmcs/openstack/main/control.pp b/modules/role/manifests/wmcs/openstack/main/control.pp index 332093b..56da43b 100644 --- a/modules/role/manifests/wmcs/openstack/main/control.pp +++ b/modules/role/manifests/wmcs/openstack/main/control.pp @@ -1,5 +1,6 @@ class role::wmcs::openstack::main::control { - include profile::openstack::main::cloudrepo - include profile::openstack::main::observerenv - include profile::openstack::main::rabbitmq + include ::profile::openstack::main::cloudrepo + include ::profile::openstack::main::clientlib + include ::profile::openstack::main::observerenv + include ::profile::openstack::main::rabbitmq } diff --git a/modules/role/manifests/wmcs/openstack/main/net.pp b/modules/role/manifests/wmcs/openstack/main/net.pp index fb2aa96..5e66fb7 100644 --- a/modules/role/manifests/wmcs/openstack/main/net.pp +++ b/modules/role/manifests/wmcs/openstack/main/net.pp @@ -1,5 +1,7 @@ class role::wmcs::openstack::main::net { include profile::openstack::main::cloudrepo - # for keystone checks which should move to control + # used by keystone checks which should move to control + # and nova-fullstack which should remain on net + include ::profile::openstack::main::clientlib include profile::openstack::main::observerenv } diff --git a/modules/role/manifests/wmcs/openstack/main/web.pp b/modules/role/manifests/wmcs/openstack/main/web.pp index 819e869..4ecd6f3 100644 --- a/modules/role/manifests/wmcs/openstack/main/web.pp +++ b/modules/role/manifests/wmcs/openstack/main/web.pp @@ -1,4 +1,5 @@ class role::wmcs::openstack::main::web { - include profile::openstack::main::observerenv - include profile::openstack::main::cloudrepo + include ::profile::openstack::main::cloudrepo + include ::profile::openstack::main::clientlib + include ::profile::openstack::main::observerenv } diff --git a/modules/role/manifests/wmcs/openstack/main/wikitech.pp b/modules/role/manifests/wmcs/openstack/main/wikitech.pp index 4e888bf..4b9eaa7 100644 --- a/modules/role/manifests/wmcs/openstack/main/wikitech.pp +++ b/modules/role/manifests/wmcs/openstack/main/wikitech.pp @@ -1,3 +1,4 @@ class role::wmcs::openstack::main::wikitech { - include profile::openstack::main::cloudrepo + include ::profile::openstack::main::cloudrepo + include ::profile::openstack::main::clientlib } diff --git a/modules/shinken/manifests/shinkengen.pp b/modules/shinken/manifests/shinkengen.pp index 8b8c0d9..a5199cb 100644 --- a/modules/shinken/manifests/shinkengen.pp +++ b/modules/shinken/manifests/shinkengen.pp @@ -4,7 +4,6 @@ # config for Shinken by hittig the wikitech API class shinken::shinkengen { include shinken - include ::openstack::clientlib package { [ 'python3-yaml', -- To view, visit https://gerrit.wikimedia.org/r/369809 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I491a04889097d286c551d2ca16d719b5efc8a604 Gerrit-PatchSet: 2 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Rush <r...@wikimedia.org> Gerrit-Reviewer: Alex Monk <kren...@gmail.com> Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org> Gerrit-Reviewer: Rush <r...@wikimedia.org> Gerrit-Reviewer: Volans <rcocci...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits