Filippo Giunchedi has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/354939 )
Change subject: Test for unreferenced files introduced by changes
......................................................................
Test for unreferenced files introduced by changes
This is a class of runtime errors not currently catched upon
compilation. Since a full puppet.git checkout is exposed as-is by the
puppet fileserver, it is possible to check for missing files in
puppet.git but referenced in catalog as "puppet://" files.
Change-Id: I20c540b27f3aff843cce16d8aaf741544c26705c
---
M puppet_compiler/controller.py
M puppet_compiler/puppet.py
2 files changed, 76 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/software/puppet-compiler
refs/changes/39/354939/1
diff --git a/puppet_compiler/controller.py b/puppet_compiler/controller.py
index ef3132d..9521818 100644
--- a/puppet_compiler/controller.py
+++ b/puppet_compiler/controller.py
@@ -192,6 +192,17 @@
return 'fail'
errors = self._compile_all()
retcode = self._make_diff(errors)
+
+ # If the change produces a catalog diff, check for files in the
catalog that are not in git.
+ # This relies on the fact a puppet.git checkout is exposed as-is by
+ # puppet fileserver, thus all files are available to hosts.
+ if retcode == 'diff':
+ try:
+ puppet.find_unreferenced_files(self.m.base_dir, self.hostname)
+ except puppet.PuppetException, e:
+ _log.exception(e)
+ return 'fail'
+
try:
self._make_output()
self._build_html(retcode)
diff --git a/puppet_compiler/puppet.py b/puppet_compiler/puppet.py
index 4ac484f..6557afb 100644
--- a/puppet_compiler/puppet.py
+++ b/puppet_compiler/puppet.py
@@ -1,7 +1,12 @@
+import json
import os
import re
import subprocess
from tempfile import SpooledTemporaryFile as spoolfile
+
+
+class PuppetException(Exception):
+ pass
def compile(hostname, basedir, vardir, *extra_flags):
@@ -61,3 +66,63 @@
temp.seek(0)
# Remove bold from term output
out.write(temp.read().replace('\x1b[1m', '').replace('\x1b[0m', ''))
+
+
+def find_unreferenced_files(basedir, hostname):
+ """
+ Find "files" introduced by the change but not present in
+ the git repo.
+
+ A "file" is defined as a resource parameter named "source" and starting
+ with "puppet:///".
+ """
+ prod_catalog = os.path.join(basedir,
+ 'production',
+ 'catalogs',
+ hostname + '.pson')
+ change_catalog = os.path.join(basedir,
+ 'change',
+ 'catalogs',
+ hostname + '.pson')
+
+ production_files = _list_catalog_files(prod_catalog)
+ change_files = _list_catalog_files(change_catalog)
+
+ if not production_files or not change_files:
+ raise PuppetException('no files found in catalogs')
+
+ files_not_found = []
+
+ for file_path in set(change_files) - set(production_files):
+ module_name, path = file_path.split('/', 5)[4:]
+ fs_path = os.path.join(basedir, 'change', 'src', 'modules',
+ module_name, 'files', path)
+ if not os.path.exists(fs_path):
+ files_not_found.append(fs_path)
+
+ if files_not_found:
+ raise PuppetException('Files introduced by change '
+ 'but not found in git: %r', files_not_found)
+
+
+def _list_catalog_files(catalog):
+ """Find all "files" (defined above) in the specified catalog."""
+ catalog_files = []
+
+ with open(catalog) as f:
+ catalog_json = json.load(f, 'latin_1')
+
+ for resource in catalog_json['data']['resources']:
+ if 'parameters' not in resource:
+ continue
+ if 'source' not in resource['parameters']:
+ continue
+ path = resource['parameters']['source']
+ if type(path) is unicode and path.startswith('puppet://'):
+ catalog_files.append(path)
+ # Some resources have a list of "source" parameters
+ if type(path) is list:
+ catalog_files.extend([x for x in path
+ if x.startswith('puppet://')])
+
+ return catalog_files
--
To view, visit https://gerrit.wikimedia.org/r/354939
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I20c540b27f3aff843cce16d8aaf741544c26705c
Gerrit-PatchSet: 1
Gerrit-Project: operations/software/puppet-compiler
Gerrit-Branch: master
Gerrit-Owner: Filippo Giunchedi <fgiunch...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
