Ottomata has uploaded a new change for review.
https://gerrit.wikimedia.org/r/280771
Change subject: [WIP] Add new scap::source define to ease bootstrapping of
repositories on deploy servers
......................................................................
[WIP] Add new scap::source define to ease bootstrapping of repositories on
deploy servers
scap::source will clone your source repo, and if set, a scap repo
expected to be at scap/$title into /srv/deployment/$title/scap.
This allows for scap/ directories to be separated from source
repositories, and allows scap repos to bootstrap themselves on
deploy servers, instead of relying on trebuchet.
eventlogging/eventbus is the guinea pig here, so this is applied to it.
Bug: T118772
Change-Id: Id0feadb4e0b274a879ec831891e52f0f70c299c1
---
A modules/eventlogging/manifests/deployment/keys.pp
M modules/eventlogging/manifests/deployment/source.pp
M modules/git/manifests/clone.pp
M modules/role/manifests/deployment/server.pp
A modules/role/manifests/eventbus/deployment/source.pp
A modules/scap/manifests/source.pp
6 files changed, 174 insertions(+), 36 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/71/280771/1
diff --git a/modules/eventlogging/manifests/deployment/keys.pp
b/modules/eventlogging/manifests/deployment/keys.pp
new file mode 100644
index 0000000..673bd72
--- /dev/null
+++ b/modules/eventlogging/manifests/deployment/keys.pp
@@ -0,0 +1,43 @@
+# == Class eventlogging::deployment::keys
+#
+# Include this class on a scap3 deployment server,
+# e.g. tin, deployment-tin, etc.
+# It sets up private keys and adds them to keyholder,
+# allowing certain groups to deploy via ssh using
+# the configured ssh key for the deploy user.
+#
+# You likely don't need to include this class directly.
+# Instead, use the eventlogging::deployment::source define.
+#
+class eventlogging::deployment::keys {
+ require ::keyholder
+ require ::keyholder::monitoring
+
+ $key_fingerprint = $::realm ? {
+ 'labs' => $::labsproject ? {
+ 'deployment-prep' =>
'02:9b:99:e2:f0:16:70:a3:d2:5a:e6:02:a3:73:0e:b0',
+ default => undef,
+ },
+ 'production' => 'b6:4e:1a:1b:4b:70:ef:91:31:cd:a3:18:9a:ca:41:44',
+ default => undef,
+ }
+
+ if !$key_fingerprint {
+ fail('Could not determine keyholder key_fingerprint for scap when
setting up eventlogging deployment source for eventlogging.')
+ }
+
+ # Use eventlogging-admins group for deployment in production,
+ # and just the current labs project group in labs.
+ $trusted_group = $::realm ? {
+ 'labs' => "project-${::labsproject}",
+ default => 'eventlogging-admins',
+ }
+
+ # For betalabs/deployment-prep, the eventlogging private key has been
+ # added to deployment-puppetmaster:/var/lib/git/private/labs/files/ssh/tin.
+ keyholder::agent { 'eventlogging':
+ trusted_group => $trusted_group,
+ key_fingerprint => $key_fingerprint,
+ key_file => 'eventlogging_rsa',
+ }
+}
diff --git a/modules/eventlogging/manifests/deployment/source.pp
b/modules/eventlogging/manifests/deployment/source.pp
index 35420e9..c9baad8 100644
--- a/modules/eventlogging/manifests/deployment/source.pp
+++ b/modules/eventlogging/manifests/deployment/source.pp
@@ -1,39 +1,27 @@
-# == Class eventlogging::deployment::source
-# Include this class on a scap3 deployment server,
-# e.g. tin, deployment-tin, etc.
-# It sets up private keys and adds them to keyholder,
-# allowing certain groups to deploy via ssh using
-# the configured ssh key for the deploy user.
+# == Define eventlogging::deployment::source
#
-class eventlogging::deployment::source {
- require ::keyholder
- require ::keyholder::monitoring
+# Sets up scap3 deployment source on a deploy server for the eventlogging
+# source repository.
+#
+# This expects that your scap directory is hosted in a repository
+# at scap/eventlogging/$title. This repository will be cloned
+# alongside of the eventlogging source repo on the deploy server.
+#
+# == Usage
+#
+# # Make sure both of 'eventlogging' and 'scap/eventlogging/eventbus'
+# # are both repositories in gerrit.
+# eventlogging::deployment::source { 'eventbus': }
+#
+define eventlogging::deployment::source()
+ include ::eventlogging::deployment::keys
- $key_fingerprint = $::realm ? {
- 'labs' => $::labsproject ? {
- 'deployment-prep' =>
'02:9b:99:e2:f0:16:70:a3:d2:5a:e6:02:a3:73:0e:b0',
- default => undef,
- },
- 'production' => 'b6:4e:1a:1b:4b:70:ef:91:31:cd:a3:18:9a:ca:41:44',
- default => undef,
- }
-
- if !$key_fingerprint {
- fail('Could not determine keyholder key_fingerprint for scap when
setting up eventlogging deployment source for eventlogging.')
- }
-
- # Use eventlogging-admins group for deployment in production,
- # and just the current labs project group in labs.
- $trusted_group = $::realm ? {
- 'labs' => "project-${::labsproject}",
- default => 'eventlogging-admins',
- }
-
- # For betalabs/deployment-prep, the eventlogging private key has been
- # added to deployment-puppetmaster:/var/lib/git/private/labs/files/ssh/tin.
- keyholder::agent { 'eventlogging':
- trusted_group => $trusted_group,
- key_fingerprint => $key_fingerprint,
- key_file => 'eventlogging_rsa',
+ # Clones the eventlogging repository into
+ # /srv/deployment/eventlogging/$title and
+ # clones the scap/eventlogging/$title repository
+ # into /srv/deployment/eventlogging/eventbus/scap
+ scap::source { "eventlogging/${title}":
+ repository => 'eventlogging',
+ recurse_submodules => true,
}
}
diff --git a/modules/git/manifests/clone.pp b/modules/git/manifests/clone.pp
index d41be3b..8a4583e 100644
--- a/modules/git/manifests/clone.pp
+++ b/modules/git/manifests/clone.pp
@@ -68,6 +68,7 @@
default => 'https://gerrit.wikimedia.org/r/p/%s.git',
}
+
$remote = $origin ? {
undef => sprintf($default_url_format, $title),
default => $origin,
diff --git a/modules/role/manifests/deployment/server.pp
b/modules/role/manifests/deployment/server.pp
index 6db95f8..f1ed23c 100644
--- a/modules/role/manifests/deployment/server.pp
+++ b/modules/role/manifests/deployment/server.pp
@@ -10,7 +10,7 @@
# NOTE: keyholder_group for role::deployment::services
# is overridden in hieradata/common/deployment/server.yaml
include role::deployment::services
- include eventlogging::deployment::source
+ include role::eventbus::deployment::source
include phabricator::deployment::source
class { 'deployment::deployment_server':
diff --git a/modules/role/manifests/eventbus/deployment/source.pp
b/modules/role/manifests/eventbus/deployment/source.pp
new file mode 100644
index 0000000..c1457b5
--- /dev/null
+++ b/modules/role/manifests/eventbus/deployment/source.pp
@@ -0,0 +1,8 @@
+# == Class role::eventbus::deployment::source
+# Configures a deploy server to deploy eventlogging source
+# for the eventbus service. Scap configs must exist
+# in the scap/eventlogging/eventbus repo.
+#
+class role::eventbus::deployment::source {
+ eventlogging::deployment::source { 'eventbus': }
+}
diff --git a/modules/scap/manifests/source.pp b/modules/scap/manifests/source.pp
new file mode 100644
index 0000000..99aa51f
--- /dev/null
+++ b/modules/scap/manifests/source.pp
@@ -0,0 +1,98 @@
+# == Define scap::source
+#
+# Sets up scap3 deployment source on a deploy server.
+# This will clone $repository at $path. If $scap_repository is set
+# it will clone it at $path/scap.
+#
+# TODO: more WIP docs!
+#
+# == Parameters
+#
+# [*package_name*]
+# Repository name in gerrit. Default: $title
+#
+# [*scap_repository_name]
+# Default: scap/$title. IF you set this to undef, a scap repo
+# will not be cloned into the scap/ directory in your source path in
+# /srv/deployment.
+#
+# [*path*]
+# Default: /srv/deployment/$title
+#
+# [*owner*]
+# Default: trebuchet
+#
+# [*group*]
+# Default: wikidev
+#
+# [*recurse_submodules*]
+# Default: false
+#
+# == Usage
+#
+# # Clones the eventlogging repository into
+# # /srv/deployment/eventlogging/eventbus and
+# # clones the scap/eventlogging/eventbus repository
+# # into /srv/deployment/eventlogging/eventbus/scap
+# scap::source { 'eventlogging/eventbus':
+# repository => 'eventlogging',
+# recurse_submodules => true,
+# }
+#
+# # Clone the 'thing/without/external/scap' repsitory into
+# # /srv/deployment/thing/without/external/scap and
+# # don't clone any scap repo.
+# scap::source { 'thing/without/external/scap':
+# scap_repository => undef,
+# }
+#
+define scap::source(
+ $repository = $title,
+ $scap_repository = "scap/${title}",
+ $path = "/srv/deployment/${title}"
+ # TODO: change scap repo owner when scap figures out
+ # how to bootstrap itself properly without trebuchet.
+ $owner = 'trebuchet'
+ $group = 'wikidev',
+ $recurse_submodules = false,
+) {
+ # We can't rely on puppet to manage arbitrary subdirectories.
+ # Use an exec to just make sure that $path's parent directories exist.
+ exec { "mkdir_scap_source_path_${title}":
+ command => "mkdir $(dirname ${path}) && chmod 775 $(dirname ${path})
&& chown ${owner}:${group} $(dirname ${path})",
+ path => '/bin:/usr/bin',
+ unless => "test -d $(dirname ${path})",
+ user => 'root',
+ }
+
+ # Clone the source repository at $path.
+ git::clone { "${repository} for ${title}":
+ # Since this define might result in multiple clones of the same
+ # $repository, it is necessary to title the git::clones with unique
+ # names. If we aren't using the repository name as the $title of
+ # git::clone, then we need to set $origin, and if provided $origin
+ # must be a full git URL. This means we can't yet use phabricator
+ # git URLs. TODO: Fix git::clone to support repository_names
+ # without specificing full git $origin URLs.
+ origin =>
"https://gerrit.wikimedia.org/r/p/${repository}.git";
+ directory => $path,
+ owner => $owner,
+ group => $group,
+ shared => true,
+ recurse_submodules => $recurse_submodules,
+ require => Exec["mkdir_scap_source_path_${title}"],
+ }
+
+ if $scap_repository {
+ # Clone the scap repository at $path/scap
+ git::clone { "${scap_repository} for ${title}":
+ origin =>
"https://gerrit.wikimedia.org/r/p/${scap_repository}.git";
+ directory => "${path}/scap",
+ owner => $owner,
+ group => $group,
+ shared => true,
+ recurse_submodules => $recurse_submodules,
+ require => Git::Clone[$repository],
+ }
+ }
+}
--
To view, visit https://gerrit.wikimedia.org/r/280771
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Id0feadb4e0b274a879ec831891e52f0f70c299c1
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
