ArielGlenn has submitted this change and it was merged.
Change subject: turn retention modules into a package
......................................................................
turn retention modules into a package
update all the import statements, add __init__.py and expect
the whole thing to be installed in a subdirectory 'retention'
mv package modules into subdir in repo as well
also update an absolute path left around from testing
Change-Id: Ife3798fc5f2c817a304c215c1158934412f0c9eb
---
M dataretention/data_auditor.py
A dataretention/retention/__init__.py
R dataretention/retention/auditor.py
R dataretention/retention/cli.py
R dataretention/retention/config.py
R dataretention/retention/examiner.py
R dataretention/retention/fileinfo.py
R dataretention/retention/magic.py
R dataretention/retention/rule.py
R dataretention/retention/runner.py
R dataretention/retention/saltclientplus.py
R dataretention/retention/status.py
R dataretention/retention/utils.py
M dataretention/rulestore.py
14 files changed, 58 insertions(+), 54 deletions(-)
Approvals:
ArielGlenn: Verified; Looks good to me, approved
diff --git a/dataretention/data_auditor.py b/dataretention/data_auditor.py
index 9db2311..739da38 100644
--- a/dataretention/data_auditor.py
+++ b/dataretention/data_auditor.py
@@ -1,10 +1,10 @@
import sys
import getopt
-sys.path.append('/home/ariel/src/wmf/git-ops-software/software/dataretention')
+sys.path.append('/srv/audits/retention/scripts/')
-from cli import CommandLine
-from auditor import FilesAuditor, LogsAuditor, HomesAuditor
-from examiner import FileExaminer, DirExaminer
+from retention.cli import CommandLine
+from retention.auditor import FilesAuditor, LogsAuditor, HomesAuditor
+from retention.examiner import FileExaminer, DirExaminer
def usage(message=None):
if message:
diff --git a/dataretention/retention/__init__.py
b/dataretention/retention/__init__.py
new file mode 100644
index 0000000..a60be23
--- /dev/null
+++ b/dataretention/retention/__init__.py
@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+'''
+Data retention library
+'''
diff --git a/dataretention/auditor.py b/dataretention/retention/auditor.py
similarity index 98%
rename from dataretention/auditor.py
rename to dataretention/retention/auditor.py
index b1230ba..e8326b6 100644
--- a/dataretention/auditor.py
+++ b/dataretention/retention/auditor.py
@@ -11,17 +11,16 @@
import zlib
import base64
-# workaround system version of magic
-sys.path = ['/srv/audits/retention/scripts'] + sys.path
+sys.path.append('/srv/audits/retention/scripts/')
-import utils
-import magic
-from status import Status
-from saltclientplus import LocalClientPlus
-from rule import Rule, RuleStore
-from config import Config
-from fileinfo import FileInfo, LogInfo, LogUtils
-from utils import JsonHelper
+import retention.utils
+import retention.magic
+from retention.status import Status
+from retention.saltclientplus import LocalClientPlus
+from retention.rule import Rule, RuleStore
+from retention.config import Config
+from retention.fileinfo import FileInfo, LogInfo, LogUtils
+from retention.utils import JsonHelper
global_keys = [key for key, value_unused in
sys.modules[__name__].__dict__.items()]
@@ -153,7 +152,7 @@
self.cutoff = Config.cf['cutoff']
- if not utils.running_locally(self.hosts_expr):
+ if not retention.utils.running_locally(self.hosts_expr):
client = LocalClientPlus()
hosts, expr_type = Runner.get_hosts_expr_type(self.hosts_expr)
self.expanded_hosts = client.cmd_expandminions(
@@ -182,10 +181,10 @@
except:
pass
- if utils.running_locally(self.hosts_expr):
+ if retention.utils.running_locally(self.hosts_expr):
self.set_up_perhost_rules()
- if not utils.running_locally(self.hosts_expr):
+ if not retention.utils.running_locally(self.hosts_expr):
self.cdb = RuleStore(self.store_filepath)
self.cdb.store_db_init(self.expanded_hosts)
self.set_up_and_export_rule_store()
@@ -195,7 +194,7 @@
self.show_ignored(Config.cf[self.locations])
self.today = time.time()
- self.magic = magic.magic_open(magic.MAGIC_NONE)
+ self.magic = retention.magic.magic_open(retention.magic.MAGIC_NONE)
self.magic.load()
self.summary = None
self.display_from_dict = FileInfo.display_from_dict
@@ -404,6 +403,7 @@
def get_perhost_rules_normal_code(self, indent):
rules = self.get_perhost_rules_as_json()
+ print "rules is", rules
code = "\n\nclass PerHostRules(object):\n" + indent + "rules = {}\n\n"
for host in rules:
@@ -445,7 +445,7 @@
def show_ignored(self, basedirs):
if self.verbose:
- if not utils.running_locally(self.hosts_expr):
+ if not retention.utils.running_locally(self.hosts_expr):
sys.stderr.write(
"INFO: The below does not include per-host rules\n")
sys.stderr.write(
@@ -1076,7 +1076,7 @@
print "WARNING: failed to load json from host"
def audit_hosts(self):
- if utils.running_locally(self.hosts_expr):
+ if retention.utils.running_locally(self.hosts_expr):
result = self.do_local_audit()
else:
result = self.runner.run_remotely()
@@ -1790,7 +1790,7 @@
code = "# -*- coding: utf-8 -*-\n"
code += self.generate_executor()
# with open(__file__, 'r') as fp_:
- with open('/home/ariel/wmf/retention-policy/data_auditor.py', 'r') as
fp_:
+ with open('/srv/audits/retention/scripts/data_auditor.py', 'r') as fp_:
code += fp_.read()
hostbatches = [self.expanded_hosts[i: i + Config.cf['batchsize']]
diff --git a/dataretention/cli.py b/dataretention/retention/cli.py
similarity index 98%
rename from dataretention/cli.py
rename to dataretention/retention/cli.py
index 77458de..6f87093 100644
--- a/dataretention/cli.py
+++ b/dataretention/retention/cli.py
@@ -8,18 +8,18 @@
import traceback
import salt.client
-sys.path.append('/home/ariel/src/wmf/git-ops-software/software/dataretention')
+sys.path.append('/srv/audits/retention/scripts/')
-from status import Status
-from rule import Rule, RuleStore
-import auditor
-from auditor import FilesAuditor, HomesAuditor, LogsAuditor
-from fileinfo import FileInfo
-import utils
-from utils import JsonHelper
-from runner import Runner
-from config import Config
-from examiner import DirExaminer, FileExaminer
+from retention.status import Status
+from retention.rule import Rule, RuleStore
+import retention.auditor
+from retention.auditor import FilesAuditor, HomesAuditor, LogsAuditor
+from retention.fileinfo import FileInfo
+import retention.utils
+from retention.utils import JsonHelper
+from retention.runner import Runner
+from retention.config import Config
+from retention.examiner import DirExaminer, FileExaminer
class LocalIgnores(object):
'''
@@ -43,7 +43,7 @@
local_ignores = {}
- if utils.running_locally(self.host):
+ if retention.utils.running_locally(self.host):
local_ignores = HomesAuditor.get_local_ignores(self.locations)
output = json.dumps(local_ignores)
print output
@@ -365,13 +365,13 @@
self.get_perhost_ignores_from_rules([host])
if Runner.running_locally(self.host):
- self.dirs_problem, self.dirs_skipped =
auditor.get_dirs_toexamine(report)
+ self.dirs_problem, self.dirs_skipped =
retention.auditor.get_dirs_toexamine(report)
else:
if host not in report:
self.dirs_problem = None
self.dirs_skipped = None
else:
- self.dirs_problem, self.dirs_skipped =
auditor.get_dirs_toexamine(report[host])
+ self.dirs_problem, self.dirs_skipped =
retention.auditor.get_dirs_toexamine(report[host])
if self.dirs_problem is None and self.dirs_skipped is None:
print "No report available from this host"
elif len(self.dirs_problem) == 0 and len(self.dirs_skipped) == 0:
diff --git a/dataretention/config.py b/dataretention/retention/config.py
similarity index 100%
rename from dataretention/config.py
rename to dataretention/retention/config.py
diff --git a/dataretention/examiner.py b/dataretention/retention/examiner.py
similarity index 96%
rename from dataretention/examiner.py
rename to dataretention/retention/examiner.py
index 187f3d4..1b31996 100644
--- a/dataretention/examiner.py
+++ b/dataretention/retention/examiner.py
@@ -3,12 +3,12 @@
import stat
import json
-sys.path.append('/home/ariel/src/wmf/git-ops-software/software/dataretention')
+sys.path.append('/srv/audits/retention/scripts/')
-from saltclientplus import LocalClientPlus
-import utils
-from utils import JsonHelper
-from fileinfo import FileInfo, EntryInfo
+from retention.saltclientplus import LocalClientPlus
+import retention.utils
+from retention.utils import JsonHelper
+from retention.fileinfo import FileInfo, EntryInfo
class FileExaminer(object):
@@ -26,7 +26,7 @@
'''
do all the work
'''
- if utils.running_locally(self.host):
+ if retention.utils.running_locally(self.host):
finf = FileInfo(self.path, None)
if finf.get_is_binary(self.num_lines):
result = "BINARY CONTENT\n"
@@ -202,7 +202,7 @@
maybe we want to fix that
'''
- if utils.running_locally(self.host):
+ if retention.utils.running_locally(self.host):
dcont = DirContents(self.path, self.batchno, self.batchsize,
self.prettyprint)
result = dcont.get_contents()
diff --git a/dataretention/fileinfo.py b/dataretention/retention/fileinfo.py
similarity index 100%
rename from dataretention/fileinfo.py
rename to dataretention/retention/fileinfo.py
diff --git a/dataretention/magic.py b/dataretention/retention/magic.py
similarity index 100%
rename from dataretention/magic.py
rename to dataretention/retention/magic.py
diff --git a/dataretention/rule.py b/dataretention/retention/rule.py
similarity index 99%
rename from dataretention/rule.py
rename to dataretention/retention/rule.py
index 2d8f53d..7b3149a 100644
--- a/dataretention/rule.py
+++ b/dataretention/retention/rule.py
@@ -4,8 +4,8 @@
import json
import traceback
import sqlite3
-from saltclientplus import LocalClientPlus
-from status import Status
+from retention.saltclientplus import LocalClientPlus
+from retention.status import Status
class Rule(object):
'''
diff --git a/dataretention/runner.py b/dataretention/retention/runner.py
similarity index 95%
rename from dataretention/runner.py
rename to dataretention/retention/runner.py
index 9349fe6..ea56d4e 100644
--- a/dataretention/runner.py
+++ b/dataretention/retention/runner.py
@@ -1,9 +1,9 @@
import sys
-sys.path.append('/home/ariel/src/wmf/git-ops-software/software/dataretention')
+sys.path.append('/srv/audits/retention/scripts/')
-from saltclientplus import LocalClientPlus
-from config import Config
+from retention.saltclientplus import LocalClientPlus
+from retention.config import Config
class Runner(object):
'''
diff --git a/dataretention/saltclientplus.py
b/dataretention/retention/saltclientplus.py
similarity index 100%
rename from dataretention/saltclientplus.py
rename to dataretention/retention/saltclientplus.py
diff --git a/dataretention/status.py b/dataretention/retention/status.py
similarity index 100%
rename from dataretention/status.py
rename to dataretention/retention/status.py
diff --git a/dataretention/utils.py b/dataretention/retention/utils.py
similarity index 100%
rename from dataretention/utils.py
rename to dataretention/retention/utils.py
diff --git a/dataretention/rulestore.py b/dataretention/rulestore.py
index c29c7b2..9490f02 100644
--- a/dataretention/rulestore.py
+++ b/dataretention/rulestore.py
@@ -6,12 +6,12 @@
import sys
import getopt
-sys.path.append('/home/ariel/src/wmf/git-ops-software/software/dataretention')
+sys.path.append('/srv/audits/retention/scripts/')
-import saltclientplus
-import utils
-from rule import Rule, RuleStore
-from status import Status
+from retention.saltclientplus import LocalClientPlus
+import retention.utils
+from retention.rule import Rule, RuleStore
+from retention.status import Status
def usage(message=None):
if message:
@@ -149,14 +149,14 @@
cdb = RuleStore(store_filepath)
cdb.store_db_init(None)
- hosts, htype = utils.get_hosts_expr_type(host)
+ hosts, htype = retention.utils.get_hosts_expr_type(host)
# if we are given one host, check that the host has a table or whine
if htype == 'glob' and '*' not in hosts:
if not Rule.check_host_table_exists(cdb, host):
usage('no such host in rule store, %s' % host)
elif htype == 'grain':
- client = saltclientplus.LocalClientPlus()
+ client = LocalClientPlus()
hosts = client.cmd_expandminions(hosts, "test.ping", expr_form=htype)
do_action(cdb, action, hosts, status, path, dryrun)
--
To view, visit https://gerrit.wikimedia.org/r/233449
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ife3798fc5f2c817a304c215c1158934412f0c9eb
Gerrit-PatchSet: 2
Gerrit-Project: operations/software
Gerrit-Branch: master
Gerrit-Owner: ArielGlenn <[email protected]>
Gerrit-Reviewer: ArielGlenn <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
