Yuvipanda has uploaded a new change for review.
https://gerrit.wikimedia.org/r/223020
Change subject: [WIP] labstore: Consolidate NFS exporting into daemon
......................................................................
[WIP] labstore: Consolidate NFS exporting into daemon
Change-Id: Ib5d55e784daa603812a97815b4ef44d9906eb2a1
---
R modules/labstore/files/nfs-exports-daemon
A modules/labstore/files/nfs-mounts.yaml
D modules/labstore/files/projects-nfs-config.yaml
M modules/labstore/lib/puppet/parser/functions/mount_nfs_volume.rb
M modules/labstore/manifests/fileserver/exports/private.pp
A modules/labstore/templates/initscripts/nfs-exports.systemd.erb
D modules/labstore/templates/initscripts/nfs-project-exports.systemd.erb
7 files changed, 321 insertions(+), 305 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/20/223020/1
diff --git a/modules/labstore/files/nfs-project-exports-daemon
b/modules/labstore/files/nfs-exports-daemon
similarity index 86%
rename from modules/labstore/files/nfs-project-exports-daemon
rename to modules/labstore/files/nfs-exports-daemon
index de7f057..6bcc779 100755
--- a/modules/labstore/files/nfs-project-exports-daemon
+++ b/modules/labstore/files/nfs-exports-daemon
@@ -94,19 +94,16 @@
return ips
-def get_projects_with_nfs(config_file_path):
+def get_public_exports(mounts_config):
+ return mounts_config['public']
+
+
+def get_projects_with_nfs(mounts_config):
"""
Returns list of populated Project objects that need NFS exports
"""
- try:
- with open(config_file_path) as f:
- projects_config = yaml.load(f)
- except:
- logging.exception('Could not load projects config file from %s',
config_file_path)
- sys.exit(1)
-
projects = []
- for name, config in projects_config.items():
+ for name, config in mounts_config['private'].items():
logging.debug('Fetching config for project %s', name)
if 'mounts' in config:
mounts = [k for k, v in config['mounts'].items()
@@ -133,10 +130,19 @@
return projects
-def sync_exports_files(projects, exports_d_base):
+def sync_exports_files(mounts_config, exports_d_base):
"""
Generate exports files for syncfs
"""
+ public_exports = get_public_exports(mounts_config)
+ for name, content in public_exports.items():
+ logging.debug('Writing exports file for public export %s', name)
+ path = os.path.join(exports_d_base, 'public_%s.exports' % name)
+ with open(path, 'w') as f:
+ f.write(content)
+ logging.info('Wrote exports file for public export %s', name)
+
+ projects = get_projects_with_nfs(mounts_config)
for project in projects:
logging.debug('Writing exports file for %s', project.name)
path = os.path.join(exports_d_base, '%s.exports' % project.name)
@@ -194,7 +200,13 @@
logging.info('Daemon starting')
while True:
- projects = get_projects_with_nfs(args.projects_config_path)
+ try:
+ with open(args.config_file_path) as f:
+ config = yaml.load(f)
+ except:
+ logging.exception('Could not load projects config file from %s',
config_file_path)
+ sys.exit(1)
+ projects = get_projects_with_nfs(config)
sync_exports_files(projects, args.exports_d_path)
if not args.dry_run:
exportfs()
diff --git a/modules/labstore/files/nfs-mounts.yaml
b/modules/labstore/files/nfs-mounts.yaml
new file mode 100644
index 0000000..8a61380
--- /dev/null
+++ b/modules/labstore/files/nfs-mounts.yaml
@@ -0,0 +1,284 @@
+public:
+ scratch: '/exp/scratch *(rw,sec=sys,sync,no_subtree_check,root_squash)'
+ root: '/exp *(ro,fsid=0,sec=sys,sync,subtree_check,root_squash,nocrossmnt)'
+private:
+ account-creation-assistance:
+ gid: 50088
+ mounts:
+ project: true
+ analytics:
+ gid: 50090
+ mounts:
+ project: true
+ bots:
+ gid: 50064
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ catgraph:
+ gid: 50588
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ cephtest:
+ gid: 52557
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ contributors:
+ gid: 50408
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ cvn:
+ gid: 50116
+ mounts:
+ project: true
+ wikidata-query:
+ gid: 52354
+ mounts:
+ home: true
+ cvresearch:
+ gid: 50118
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ deployment-prep:
+ gid: 50120
+ mounts:
+ project: true
+ dumps:
+ gid: 50124
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ dwl:
+ gid: 50815
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ editor-engagement:
+ gid: 50068
+ mounts:
+ project: true
+ fastcci:
+ gid: 50983
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ fatg:
+ gid: 52291
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ gsoc2014-fonttailor-demo:
+ gid: 52253
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ huggle:
+ gid: 50168
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ ircnotifier:
+ gid: 52361
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ maps:
+ gid: 50196
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ maps-team:
+ gid: 52497
+ mounts:
+ scratch: true
+ math:
+ gid: 50398
+ mounts:
+ project: true
+ scratch: true
+ megacron:
+ gid: 51769
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ mwoffliner:
+ gid: 52376
+ mounts:
+ scratch: true
+ openstack:
+ gid: 50076
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ orgcharts:
+ gid: 50228
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ pdbhandler:
+ gid: 50240
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ project-proxy:
+ gid: 50254
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ puppet3-diffs:
+ gid: 51904
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ quarry:
+ gid: 52243
+ mounts:
+ project: true
+ snuggle:
+ gid: 50470
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ sugarcrm:
+ gid: 50292
+ mounts:
+ project: true
+ testlabs:
+ gid: 50302
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ tools:
+ gid: 50380
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ toolsbeta:
+ gid: 50610
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ toolserver-legacy:
+ gid: 52379
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ utrs:
+ gid: 50318
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ video:
+ gid: 52318
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ wikidata-dev:
+ gid: 50084
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ wikidata-quality:
+ gid: 52377
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ wikidata-topicmaps:
+ gid: 51980
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ wikidumpparse:
+ gid: 51824
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ wikisource-tools:
+ gid: 50336
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ wikispy:
+ gid: 52518
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ wikiviajesve:
+ gid: 52059
+ mounts:
+ dumps: true
+ home: true
+ project: true
+ scratch: true
+ wmt:
+ gid: 52488
+ mounts:
+ project: true
diff --git a/modules/labstore/files/projects-nfs-config.yaml
b/modules/labstore/files/projects-nfs-config.yaml
deleted file mode 100644
index 2ffff47..0000000
--- a/modules/labstore/files/projects-nfs-config.yaml
+++ /dev/null
@@ -1,280 +0,0 @@
-account-creation-assistance:
- gid: 50088
- mounts:
- project: true
-analytics:
- gid: 50090
- mounts:
- project: true
-bots:
- gid: 50064
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-catgraph:
- gid: 50588
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-cephtest:
- gid: 52557
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-contributors:
- gid: 50408
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-cvn:
- gid: 50116
- mounts:
- project: true
-wikidata-query:
- gid: 52354
- mounts:
- home: true
-cvresearch:
- gid: 50118
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-deployment-prep:
- gid: 50120
- mounts:
- project: true
-dumps:
- gid: 50124
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-dwl:
- gid: 50815
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-editor-engagement:
- gid: 50068
- mounts:
- project: true
-fastcci:
- gid: 50983
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-fatg:
- gid: 52291
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-gsoc2014-fonttailor-demo:
- gid: 52253
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-huggle:
- gid: 50168
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-ircnotifier:
- gid: 52361
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-maps:
- gid: 50196
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-maps-team:
- gid: 52497
- mounts:
- scratch: true
-math:
- gid: 50398
- mounts:
- project: true
- scratch: true
-megacron:
- gid: 51769
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-mwoffliner:
- gid: 52376
- mounts:
- scratch: true
-openstack:
- gid: 50076
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-orgcharts:
- gid: 50228
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-pdbhandler:
- gid: 50240
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-project-proxy:
- gid: 50254
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-puppet3-diffs:
- gid: 51904
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-quarry:
- gid: 52243
- mounts:
- project: true
-snuggle:
- gid: 50470
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-sugarcrm:
- gid: 50292
- mounts:
- project: true
-testlabs:
- gid: 50302
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-tools:
- gid: 50380
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-toolsbeta:
- gid: 50610
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-toolserver-legacy:
- gid: 52379
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-utrs:
- gid: 50318
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-video:
- gid: 52318
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-wikidata-dev:
- gid: 50084
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-wikidata-quality:
- gid: 52377
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-wikidata-topicmaps:
- gid: 51980
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-wikidumpparse:
- gid: 51824
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-wikisource-tools:
- gid: 50336
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-wikispy:
- gid: 52518
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-wikiviajesve:
- gid: 52059
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
-wmt:
- gid: 52488
- mounts:
- project: true
diff --git a/modules/labstore/lib/puppet/parser/functions/mount_nfs_volume.rb
b/modules/labstore/lib/puppet/parser/functions/mount_nfs_volume.rb
index bd9e2a6..235e471 100644
--- a/modules/labstore/lib/puppet/parser/functions/mount_nfs_volume.rb
+++ b/modules/labstore/lib/puppet/parser/functions/mount_nfs_volume.rb
@@ -12,7 +12,7 @@
@@labs_nfs_config = nil
newfunction(:mount_nfs_volume, :type => :rvalue, :arity => 2) do |args|
module_path = function_get_module_path(['labstore'])
- path = "#{module_path}/files/projects-nfs-config.yaml"
+ path = "#{module_path}/files/nfs-mounts.yaml"
mtime = File.stat(path).mtime
if @@labs_nfs_config_touched.nil? || mtime != @@labs_nfs_config_touched
@@labs_nfs_config = function_loadyaml([path])
diff --git a/modules/labstore/manifests/fileserver/exports/private.pp
b/modules/labstore/manifests/fileserver/exports/private.pp
index 65e9538..ef4ff95 100644
--- a/modules/labstore/manifests/fileserver/exports/private.pp
+++ b/modules/labstore/manifests/fileserver/exports/private.pp
@@ -25,22 +25,22 @@
source => 'puppet:///modules/labstore/sync-exports',
}
- file { '/etc/projects-nfs-config.yaml':
+ file { '/etc/nfs-mounts.yaml':
owner => 'root',
group => 'root',
mode => '0444',
- source => 'puppet:///modules/labstore/projects-nfs-config.yaml',
+ source => 'puppet:///modules/labstore/nfs-mounts.yaml',
}
- file { '/usr/local/bin/nfs-project-exports-daemon':
+ file { '/usr/local/bin/nfs-exports-daemon':
owner => 'root',
group => 'root',
mode => '0555',
- source => 'puppet:///modules/labstore/nfs-project-exports-daemon',
- notify => Service['nfs-project-exports'],
+ source => 'puppet:///modules/labstore/nfs-exports-daemon',
+ notify => Service['nfs-exports'],
}
- base::service_unit { 'nfs-project-exports':
+ base::service_unit { 'nfs-exports':
ensure => present,
systemd => true,
}
diff --git a/modules/labstore/templates/initscripts/nfs-exports.systemd.erb
b/modules/labstore/templates/initscripts/nfs-exports.systemd.erb
new file mode 100644
index 0000000..32e4efd
--- /dev/null
+++ b/modules/labstore/templates/initscripts/nfs-exports.systemd.erb
@@ -0,0 +1,7 @@
+[Unit]
+Description=Daemon to do private NFS project exports
+
+[Service]
+ExecStart=/usr/local/bin/nfs-exports-daemon --exports-d-path /etc/exports.d/
--projects-config-path /etc/nfs-mounts.yaml
+Restart=on-failure
+User=nfsmanager
diff --git
a/modules/labstore/templates/initscripts/nfs-project-exports.systemd.erb
b/modules/labstore/templates/initscripts/nfs-project-exports.systemd.erb
deleted file mode 100644
index 1197605..0000000
--- a/modules/labstore/templates/initscripts/nfs-project-exports.systemd.erb
+++ /dev/null
@@ -1,7 +0,0 @@
-[Unit]
-Description=Daemon to do private NFS project exports
-
-[Service]
-ExecStart=/usr/local/bin/nfs-project-exports-daemon --exports-d-path
/etc/exports.d/ --projects-config-path /etc/projects-nfs-config.yaml
-Restart=on-failure
-User=nfsmanager
--
To view, visit https://gerrit.wikimedia.org/r/223020
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib5d55e784daa603812a97815b4ef44d9906eb2a1
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Yuvipanda <yuvipa...@gmail.com>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
