Andrew Bogott has submitted this change and it was merged.
Change subject: download: convert into a module and clean up
......................................................................
download: convert into a module and clean up
Change-Id: Ied0f7e3bd696b0fb35bf2d705b8e0dda62c51879
---
D manifests/misc/download.pp
M manifests/site.pp
R modules/download/files/.pep8
R modules/download/files/apache/download.mediawiki.org
R modules/download/files/exports
R modules/download/files/gluster/gluster-rsync-cron.sh
R modules/download/files/gluster/wmfdumpsmirror.py
R modules/download/files/lighttpd.conf
R modules/download/files/rsync-dumps.sh
R modules/download/files/rsync/rsync.default.downloadmirror
R modules/download/files/rsync/rsync.default.downloadprimary
R modules/download/files/rsync/rsyncd.conf.downloadmirror
R modules/download/files/rsync/rsyncd.conf.downloadprimary
A modules/download/manifests/cron-rsync-dumps.pp
A modules/download/manifests/gluster.pp
A modules/download/manifests/init.pp
A modules/download/manifests/kiwix.pp
A modules/download/manifests/mediawiki.pp
A modules/download/manifests/mirror.pp
A modules/download/manifests/primary.pp
A modules/download/manifests/wikimedia.pp
21 files changed, 219 insertions(+), 239 deletions(-)
Approvals:
Andrew Bogott: Looks good to me, approved
jenkins-bot: Verified
diff --git a/manifests/misc/download.pp b/manifests/misc/download.pp
deleted file mode 100644
index a34d027..0000000
--- a/manifests/misc/download.pp
+++ /dev/null
@@ -1,231 +0,0 @@
-# misc::download
-
-class misc::download::cron-rsync-dumps {
- file { '/usr/local/bin/rsync-dumps.sh':
- mode => 0755,
- owner => root,
- group => root,
- path => '/usr/local/bin/rsync-dumps.sh',
- source => 'puppet:///files/misc/scripts/rsync-dumps.sh';
- }
-
- cron { 'rsync-dumps':
- ensure => present,
- command => '/usr/local/bin/rsync-dumps.sh',
- user => root,
- minute => '0',
- hour => '*/2',
- require => File['/usr/local/bin/rsync-dumps.sh'];
- }
-}
-
-class misc::download-wikimedia {
- system::role { "misc::download-wikimedia": description =>
"download.wikimedia.org" }
-
- package { lighttpd:
- ensure => latest;
- }
-
- file {
- "/etc/lighttpd/lighttpd.conf":
- mode => 0444,
- owner => root,
- group => root,
- path => "/etc/lighttpd/lighttpd.conf",
- source => "puppet:///files/download/lighttpd.conf";
- }
-
- service { lighttpd:
- ensure => running;
- }
-
- package { nfs-kernel-server:
- ensure => present;
- }
-
- file { "/etc/exports":
- require => Package[nfs-kernel-server],
- mode => 0444,
- owner => root,
- group => root,
- source => "puppet:///files/download/exports",
- }
-
- service { nfs-kernel-server:
- require => [ Package[nfs-kernel-server], File["/etc/exports"] ],
- }
-
- package { 'gnupg':
- ensure => latest;
- }
-
- include generic::higher_min_free_kbytes
-
- monitor_service { "lighttpd http": description => "Lighttpd HTTP",
check_command => "check_http" }
- monitor_service { "nfs": description => "NFS", check_command =>
"check_tcp!2049" }
-}
-
-class misc::download-primary {
- system::role { "misc::download-primary": description => "Service for
rsync to internal download mirrors" }
-
- include role::mirror::common
-
- file {
- "/etc/rsyncd.conf":
- require => Package[rsync],
- mode => 0444,
- owner => root,
- group => root,
- source =>
"puppet:///files/rsync/rsyncd.conf.downloadprimary";
- "/etc/default/rsync":
- require => Package[rsync],
- mode => 0444,
- owner => root,
- group => root,
- source =>
"puppet:///files/rsync/rsync.default.downloadprimary";
- }
-
- service { rsync:
- require => [ Package[rsync], File["/etc/rsyncd.conf"],
File["/etc/default/rsync"] ],
- ensure => running;
- }
-}
-
-class misc::download-mirror {
- system::role { "misc::download-mirror": description => "Service for
rsync to external download mirrors" }
-
- include role::mirror::common
-
- file {
- "/etc/rsyncd.conf":
- require => Package[rsync],
- mode => 0444,
- owner => root,
- group => root,
- source =>
"puppet:///files/rsync/rsyncd.conf.downloadmirror";
- "/etc/default/rsync":
- require => Package[rsync],
- mode => 0444,
- owner => root,
- group => root,
- source =>
"puppet:///files/rsync/rsync.default.downloadmirror";
- }
-
- service { rsync:
- require => [ Package[rsync], File["/etc/rsyncd.conf"],
File["/etc/default/rsync"] ],
- ensure => running;
- }
-}
-
-
-class misc::download-mediawiki {
-
- system::role { "misc::download-mediawiki": description => "MediaWiki
download" }
-
- # FIXME: require apache
-
- file {
- #apache config
- "/etc/apache2/sites-available/download.mediawiki.org":
- mode => 0444,
- owner => root,
- group => root,
- source =>
"puppet:///files/apache/sites/download.mediawiki.org";
- "/srv/org/mediawiki":
- owner => "root",
- group => "root",
- mode => 0775,
- ensure => directory;
- "/srv/org/mediawiki/download":
- owner => "mwdeploy",
- group => "mwdeploy",
- mode => 0775,
- ensure => directory;
- }
-
- apache_site { "download.mediawiki.org": name =>
"download.mediawiki.org" }
-}
-
-class misc::download-gluster {
- include role::mirror::common
- include gluster::client
-
- system::role { "misc::download-gluster": description => "Gluster dumps
copy" }
-
- file {
- '/mnt/glusterpublicdata':
- ensure => directory,
- owner => "root",
- group => "root",
- mode => 0775;
- }
-
- mount {
- '/mnt/glusterpublicdata':
- ensure => present,
- device => 'labstore1.pmtpa.wmnet:/publicdata-project',
- fstype => 'glusterfs',
- options =>
'defaults,_netdev=bond0,log-level=WARNING,log-file=/var/log/gluster.log',
- require => [Package['glusterfs-client'],
File['/mnt/glusterpublicdata']];
- }
-
- file {
- '/usr/local/bin/wmfdumpsmirror.py':
- ensure => present,
- mode => '0755',
- source => 'puppet:///files/mirror/wmfdumpsmirror.py';
- '/usr/local/sbin/gluster-rsync-cron.sh':
- ensure => present,
- mode => '0755',
- source =>
'puppet:///files/mirror/gluster-rsync-cron.sh',
- }
-
- cron {
- 'dumps_gluster_rsync':
- ensure => present,
- user => root,
- minute => '50',
- hour => '3',
- command => '/usr/local/sbin/gluster-rsync-cron.sh',
- environment => '[email protected]',
- require => [ File[
['/usr/local/bin/wmfdumpsmirror.py', '/usr/local/sbin/gluster-rsync-cron.sh'],
- ['/mnt/glusterpublicdata'] ] ];
- }
-}
-
-
-class misc::kiwix-mirror {
- # TODO: add system::role
-
- group { mirror:
- ensure => "present";
- }
-
- user { mirror:
- name => "mirror",
- gid => "mirror",
- groups => [ "www-data"],
- membership => "minimum",
- home => "/data/home",
- shell => "/bin/bash";
- }
-
- file {
- "/data/xmldatadumps/public/kiwix":
- ensure => "/data/xmldatadumps/public/other/kiwix";
- "/data/xmldatadumps/public/other/kiwix":
- owner => "mirror",
- group => "mirror",
- mode => 0644,
- ensure => present;
- }
-
- cron { kiwix-mirror-update:
- command => "rsync -vzrlptD
download.kiwix.org::download.kiwix.org/zim/0.9/
/data/xmldatadumps/public/other/kiwix/zim/0.9/ >/dev/null 2>&1",
- user => mirror,
- minute => '*/15',
- ensure => present;
- }
-
-}
-
diff --git a/manifests/site.pp b/manifests/site.pp
index 13faf01..e2bf2ad 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -533,10 +533,9 @@
admins::roots,
groups::wikidev,
accounts::catrope,
- misc::download-wikimedia,
- misc::download-primary,
- misc::download::cron-rsync-dumps,
- misc::kiwix-mirror
+ download,
+ download::primary,
+ download::kiwix
}
node "dataset1001.wikimedia.org" {
@@ -547,10 +546,9 @@
admins::roots,
groups::wikidev,
accounts::catrope,
- misc::download-wikimedia,
- misc::download-mirror,
- misc::download::cron-rsync-dumps,
- misc::download-gluster
+ download,
+ download::mirror,
+ download::gluster
}
# pmtpa dbs
diff --git a/files/mirror/.pep8 b/modules/download/files/.pep8
similarity index 100%
rename from files/mirror/.pep8
rename to modules/download/files/.pep8
diff --git a/files/apache/sites/download.mediawiki.org
b/modules/download/files/apache/download.mediawiki.org
similarity index 100%
rename from files/apache/sites/download.mediawiki.org
rename to modules/download/files/apache/download.mediawiki.org
diff --git a/files/download/exports b/modules/download/files/exports
similarity index 100%
rename from files/download/exports
rename to modules/download/files/exports
diff --git a/files/mirror/gluster-rsync-cron.sh
b/modules/download/files/gluster/gluster-rsync-cron.sh
similarity index 100%
rename from files/mirror/gluster-rsync-cron.sh
rename to modules/download/files/gluster/gluster-rsync-cron.sh
diff --git a/files/mirror/wmfdumpsmirror.py
b/modules/download/files/gluster/wmfdumpsmirror.py
similarity index 100%
rename from files/mirror/wmfdumpsmirror.py
rename to modules/download/files/gluster/wmfdumpsmirror.py
diff --git a/files/download/lighttpd.conf b/modules/download/files/lighttpd.conf
similarity index 100%
rename from files/download/lighttpd.conf
rename to modules/download/files/lighttpd.conf
diff --git a/files/misc/scripts/rsync-dumps.sh
b/modules/download/files/rsync-dumps.sh
similarity index 100%
rename from files/misc/scripts/rsync-dumps.sh
rename to modules/download/files/rsync-dumps.sh
diff --git a/files/rsync/rsync.default.downloadmirror
b/modules/download/files/rsync/rsync.default.downloadmirror
similarity index 100%
rename from files/rsync/rsync.default.downloadmirror
rename to modules/download/files/rsync/rsync.default.downloadmirror
diff --git a/files/rsync/rsync.default.downloadprimary
b/modules/download/files/rsync/rsync.default.downloadprimary
similarity index 100%
rename from files/rsync/rsync.default.downloadprimary
rename to modules/download/files/rsync/rsync.default.downloadprimary
diff --git a/files/rsync/rsyncd.conf.downloadmirror
b/modules/download/files/rsync/rsyncd.conf.downloadmirror
similarity index 100%
rename from files/rsync/rsyncd.conf.downloadmirror
rename to modules/download/files/rsync/rsyncd.conf.downloadmirror
diff --git a/files/rsync/rsyncd.conf.downloadprimary
b/modules/download/files/rsync/rsyncd.conf.downloadprimary
similarity index 100%
rename from files/rsync/rsyncd.conf.downloadprimary
rename to modules/download/files/rsync/rsyncd.conf.downloadprimary
diff --git a/modules/download/manifests/cron-rsync-dumps.pp
b/modules/download/manifests/cron-rsync-dumps.pp
new file mode 100644
index 0000000..e58b6ba
--- /dev/null
+++ b/modules/download/manifests/cron-rsync-dumps.pp
@@ -0,0 +1,18 @@
+class download::cron-rsync-dumps {
+ file { '/usr/local/bin/rsync-dumps.sh':
+ mode => '0755',
+ owner => root,
+ group => root,
+ path => '/usr/local/bin/rsync-dumps.sh',
+ source => 'puppet:///modules/download/files/rsync-dumps.sh';
+ }
+
+ cron { 'rsync-dumps':
+ ensure => present,
+ command => '/usr/local/bin/rsync-dumps.sh',
+ user => root,
+ minute => '0',
+ hour => '*/2',
+ require => File['/usr/local/bin/rsync-dumps.sh'];
+ }
+}
diff --git a/modules/download/manifests/gluster.pp
b/modules/download/manifests/gluster.pp
new file mode 100644
index 0000000..a6dba38
--- /dev/null
+++ b/modules/download/manifests/gluster.pp
@@ -0,0 +1,36 @@
+class download::gluster {
+ include role::mirror::common
+ include generic::gluster-client
+
+ system_role { "download-gluster": description => "Gluster dumps copy" }
+
+ mount { '/mnt/glusterpublicdata':
+ ensure => mounted,
+ device => 'labstore1.pmtpa.wmnet:/publicdata-project',
+ fstype => 'glusterfs',
+ options =>
'defaults,_netdev=bond0,log-level=WARNING,log-file=/var/log/gluster.log',
+ require => Package['glusterfs-client'],
+ }
+
+ file { '/usr/local/bin/wmfdumpsmirror.py':
+ ensure => present,
+ mode => '0755',
+ source => 'puppet:///modules/download/files/gluster/wmfdumpsmirror.py',
+ }
+
+ file{ '/usr/local/sbin/gluster-rsync-cron.sh':
+ ensure => present,
+ mode => '0755',
+ source =>
'puppet:///modules/download/files/gluster/gluster-rsync-cron.sh',
+ }
+
+ cron { 'dumps_gluster_rsync':
+ ensure => present,
+ user => root,
+ minute => '50',
+ hour => '3',
+ command => '/usr/local/sbin/gluster-rsync-cron.sh',
+ environment => '[email protected]',
+ require => [ File[
['/usr/local/bin/wmfdumpsmirror.py','/usr/local/sbin/gluster-rsync-cron.sh'] ],
Mount['/mnt/glusterpublicdata'] ]
+ }
+}
diff --git a/modules/download/manifests/init.pp
b/modules/download/manifests/init.pp
new file mode 100644
index 0000000..6cd9207
--- /dev/null
+++ b/modules/download/manifests/init.pp
@@ -0,0 +1,4 @@
+class download {
+ include download::cron-rsync-dumps,
+ download::wikimedia
+}
diff --git a/modules/download/manifests/kiwix.pp
b/modules/download/manifests/kiwix.pp
new file mode 100644
index 0000000..2d3240d
--- /dev/null
+++ b/modules/download/manifests/kiwix.pp
@@ -0,0 +1,30 @@
+class download::kiwix {
+ # TODO: add system_role
+ group { 'mirror':
+ ensure => present,
+ }
+
+ user { 'mirror':
+ name => mirror,
+ gid => mirror,
+ groups => www-data,
+ membership => minimum,
+ home => '/data/home',
+ shell => '/bin/bash',
+ }
+
+ file { '/data/xmldatadumps/public/kiwix':
+ ensure => link,
+ target => '/data/xmldatadumps/public/other/kiwix',
+ owner => mirror,
+ group => mirror,
+ mode => '0644',
+ }
+
+ cron { 'kiwix-mirror-update':
+ ensure => present,
+ command => "rsync -vzrlptD
download.kiwix.org::download.kiwix.org/zim/0.9/
/data/xmldatadumps/public/other/kiwix/zim/0.9/ >/dev/null 2>&1",
+ user => mirror,
+ minute => '*/15',
+ }
+}
diff --git a/modules/download/manifests/mediawiki.pp
b/modules/download/manifests/mediawiki.pp
new file mode 100644
index 0000000..02fb361
--- /dev/null
+++ b/modules/download/manifests/mediawiki.pp
@@ -0,0 +1,31 @@
+class download::mediawiki {
+
+ system_role { "download-mediawiki": description => "MediaWiki download" }
+
+ package { 'apache':
+ ensure => present,
+ }
+
+ file { '/etc/apache2/sites-available/download.mediawiki.org':
+ mode => '0444',
+ owner => root,
+ group => root,
+ source => 'puppet:///modules/downloads/apache/download.mediawiki.org',
+ }
+
+ file { '/srv/org/mediawiki':
+ ensure => directory,
+ owner => root,
+ group => root,
+ mode => '0775',
+ }
+
+ file { '/srv/org/mediawiki/download':
+ ensure => directory,
+ owner => mwdeploy,
+ group => mwdeploy,
+ mode => '0775',
+ }
+
+ apache_site { "download.mediawiki.org": name => "download.mediawiki.org" }
+}
diff --git a/modules/download/manifests/mirror.pp
b/modules/download/manifests/mirror.pp
new file mode 100644
index 0000000..617c7a1
--- /dev/null
+++ b/modules/download/manifests/mirror.pp
@@ -0,0 +1,27 @@
+class download::mirror {
+ system_role { "download-mirror": description => "Service for rsync to
external download mirrors" }
+
+ include role::mirror::common
+
+ file { '/etc/rsyncd.conf':
+ mode => '0444',
+ owner => root,
+ group => root,
+ source =>
'puppet:///modules/download/files/rsync/rsyncd.conf.downloadmirror',
+ require => Package['rsync'],
+ }
+
+ file { '/etc/default/rsync':
+ mode => '0444',
+ owner => root,
+ group => root,
+ source =>
'puppet:///modules/download/files/rsync/rsync.default.downloadmirror',
+ require => Package['rsync'],
+ }
+
+ service { 'rsync':
+ ensure => running,
+ require => [ Package['rsync'], File['/etc/rsyncd.conf'],
+File['/etc/default/rsync'] ],
+ }
+}
diff --git a/modules/download/manifests/primary.pp
b/modules/download/manifests/primary.pp
new file mode 100644
index 0000000..d0637f5
--- /dev/null
+++ b/modules/download/manifests/primary.pp
@@ -0,0 +1,27 @@
+class download::primary {
+ system_role { "download-primary": description => "Service for rsync to
internal download mirrors" }
+
+ include role::mirror::common
+
+ file { '/etc/rsyncd.conf':
+ mode => '0444',
+ owner => root,
+ group => root,
+ source =>
'puppet:///modules/download/files/rsync/rsyncd.conf.downloadprimary',
+ require => Package['rsync'],
+ }
+
+ file { '/etc/default/rsync':
+ mode => '0444',
+ owner => root,
+ group => root,
+ source
=>'puppet:///modules/download/files/rsync/rsync.default.downloadprimary',
+ require => Package['rsync'],
+ }
+
+ service { 'rsync':
+ ensure => running,
+ require => [ Package['rsync'], File['/etc/rsyncd.conf'],
+File['/etc/default/rsync'] ],
+ }
+}
diff --git a/modules/download/manifests/wikimedia.pp
b/modules/download/manifests/wikimedia.pp
new file mode 100644
index 0000000..0817ba7
--- /dev/null
+++ b/modules/download/manifests/wikimedia.pp
@@ -0,0 +1,40 @@
+class download::wikimedia {
+ system_role { "download-wikimedia": description =>
"download.wikimedia.org" }
+
+ package { 'lighttpd':
+ ensure => latest;
+ }
+
+ file { '/etc/lighttpd/lighttpd.conf':
+ mode => '0444',
+ owner => root,
+ group => root,
+ path => '/etc/lighttpd/lighttpd.conf',
+ source => 'puppet:///modules/download/files/lighttpd.conf',
+ }
+
+ service { 'lighttpd':
+ ensure => running;
+ }
+
+ package { 'nfs-kernel-server':
+ ensure => present;
+ }
+
+ file { '/etc/exports':
+ mode => '0444',
+ owner => root,
+ group => root,
+ source => 'puppet:///modules/download/files/exports',
+ require => package['nfs-kernel-server'],
+ }
+
+ service { 'nfs-kernel-server':
+ require => [ Package['nfs-kernel-server'], File['/etc/exports'] ],
+ }
+
+ include generic::higher_min_free_kbytes
+
+ monitor_service { "lighttpd http": description => "Lighttpd HTTP",
check_command => "check_http" }
+ monitor_service { "nfs": description => "NFS", check_command =>
"check_tcp!2049" }
+}
--
To view, visit https://gerrit.wikimedia.org/r/95460
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ied0f7e3bd696b0fb35bf2d705b8e0dda62c51879
Gerrit-PatchSet: 2
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Matanya <[email protected]>
Gerrit-Reviewer: Andrew Bogott <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
