ArielGlenn has submitted this change and it was merged.
Change subject: convert dir examiner to use salt module
......................................................................
convert dir examiner to use salt module
next up: cli must be converted
Change-Id: Idab4a867e4bac7e6e8e433ea2497a04c34edc34d
---
M dataretention/data_auditor.py
M dataretention/retention/cli.py
M dataretention/retention/examiner.py
M dataretention/retention/retentionaudit.py
4 files changed, 99 insertions(+), 76 deletions(-)
Approvals:
ArielGlenn: Verified; Looks good to me, approved
diff --git a/dataretention/data_auditor.py b/dataretention/data_auditor.py
index a987360..07d23a5 100644
--- a/dataretention/data_auditor.py
+++ b/dataretention/data_auditor.py
@@ -7,7 +7,7 @@
from retention.remotefileauditor import RemoteFilesAuditor
from retention.remotelogauditor import RemoteLogsAuditor
from retention.remotehomeauditor import RemoteHomesAuditor
-from retention.examiner import RemoteFileExaminer, DirExaminer
+from retention.examiner import RemoteFileExaminer, RemoteDirExaminer
def usage(message=None):
if message:
@@ -209,7 +209,7 @@
if dir_info is not None:
# for now more than 1000 entries in a dir = we silently toss them
- direxam = DirExaminer(dir_info, hosts_expr, batchno, 1000, timeout)
+ direxam = RemoteDirExaminer(dir_info, hosts_expr, batchno, 1000,
timeout)
direxam.run()
sys.exit(0)
elif file_info is not None:
diff --git a/dataretention/retention/cli.py b/dataretention/retention/cli.py
index f3f8863..617b370 100644
--- a/dataretention/retention/cli.py
+++ b/dataretention/retention/cli.py
@@ -19,7 +19,7 @@
import retention.utils
from retention.utils import JsonHelper
from retention.config import Config
-from retention.examiner import DirExaminer, RemoteFileExaminer
+from retention.examiner import RemoteDirExaminer, RemoteFileExaminer
import retention.fileutils
import retention.ruleutils
@@ -549,7 +549,7 @@
# fixme batchno? batchno should increment too
# for now more than 1000 entries in a dir = we silently toss them
- direxamin = DirExaminer(path, self.host, batchno, 1000, self.timeout,
prettyprint=False)
+ direxamin = RemoteDirExaminer(path, self.host, batchno, 1000,
self.timeout, prettyprint=False)
contents = direxamin.run(True)
if contents is not None:
contents = contents.split("\n")
diff --git a/dataretention/retention/examiner.py
b/dataretention/retention/examiner.py
index 129fc92..cd559c8 100644
--- a/dataretention/retention/examiner.py
+++ b/dataretention/retention/examiner.py
@@ -2,6 +2,7 @@
import sys
import stat
import json
+import logging
sys.path.append('/srv/audits/retention/scripts/')
@@ -9,6 +10,8 @@
import retention.utils
from retention.utils import JsonHelper
from retention.fileinfo import FileInfo, EntryInfo
+
+log = logging.getLogger(__name__)
class RemoteFileExaminer(object):
@@ -164,24 +167,28 @@
def display_json(self, json_text):
if not self.prettyprint:
print json_text
- return
+ return json_text
try:
item = json.loads(json_text, object_hook=JsonHelper.decode_dict)
except:
print json_text
- return
+ return json_text
output = FileInfo.format_pretty_output_from_dict(item, path_justify=50)
print output
+ return output
def show_batch(self):
+ output = []
for entry in self.batch_entryinfo:
- self.display_json(entry)
+ output.append(self.display_json(entry))
+ output = '\n'.join(output)
+ return output
+
-
-class DirExaminer(object):
+class RemoteDirExaminer(object):
'''
- retrieval and display of directory contents on local or remote host
+ retrieval and display of directory contents on remote host
'''
def __init__(self, path, host, batchno=1, batchsize=300, timeout=20,
prettyprint=False):
@@ -202,81 +209,89 @@
maybe we want to fix that
'''
- if retention.utils.running_locally(self.host):
- dcont = DirContents(self.path, self.batchno, self.batchsize,
- self.prettyprint)
- result = dcont.get_contents()
- if result != 'ok':
- print ('WARNING: failed to get directory contents'
- 'for <%s> (%s)'
- % (self.path, result))
- else:
- dcont.get_batch_entryinfo()
- dcont.show_batch()
- else:
- while True:
- client = LocalClientPlus()
- code = "# -*- coding: utf-8 -*-\n"
- code += self.generate_executor()
- with open(__file__, 'r') as fp_:
- code += fp_.read()
- result = client.cmd([self.host], "cmd.exec_code",
- ["python2", code],
- expr_form='list',
- timeout=self.timeout)
- if self.host in result:
- lines = result[self.host].split("\n")
+ while True:
+ client = LocalClientPlus()
+ module_args = [self.path, self.batchno,
+ self.batchsize, self.timeout,
+ quiet]
- maxlen = 0
+ result = client.cmd([self.host],
+ "retentionaudit.examine_dir",
+ module_args, expr_form='list',
+ timeout=self.timeout)
+
+ if self.host in result:
+ lines = result[self.host].split("\n")
+
+ maxlen = 0
+ for line in lines:
+ if (line.startswith("WARNING:") or
+ line.startswith("INFO:")):
+ continue
+ else:
+ try:
+ entry = json.loads(
+ line, object_hook=JsonHelper.decode_dict)
+ if len(entry['path']) > maxlen:
+ maxlen = len(entry['path'])
+ except:
+ continue
+
+ if not quiet:
for line in lines:
if (line.startswith("WARNING:") or
- line.startswith("INFO:")):
- continue
+ line.startswith("INFO:")):
+ print line
else:
try:
entry = json.loads(
- line, object_hook=JsonHelper.decode_dict)
- if len(entry['path']) > maxlen:
- maxlen = len(entry['path'])
+ line,
+ object_hook=JsonHelper.decode_dict)
+ EntryInfo.display_from_dict(
+ entry, True, maxlen)
except:
- continue
-
- if not quiet:
- for line in lines:
- if (line.startswith("WARNING:") or
- line.startswith("INFO:")):
print line
- else:
- try:
- entry = json.loads(
- line,
- object_hook=JsonHelper.decode_dict)
- EntryInfo.display_from_dict(
- entry, True, maxlen)
- except:
- print line
- return result[self.host]
- else:
- print "Failed to retrieve dir content for", self.path,
"on", self.host
- continuing = ("Try again? Y/N [N]: ")
- if continuing == "":
- continuing = "N"
- if continuing.upper() != "Y":
- return None
+ return result[self.host]
+ else:
+ print "Failed to retrieve dir content for", self.path, "on",
self.host
+ continuing = ("Try again? Y/N [N]: ")
+ if continuing == "":
+ continuing = "N"
+ if continuing.upper() != "Y":
+ return None
- def generate_executor(self):
- '''
- horrible hack: this code is fed to salt when we feed this
- script to stdin to run it remotely, thus bypassing all
- the command line argument parsing logic
- in this case we set up for DirExaminer directly
+class LocalDirExaminer(object):
+ '''
+ retrieval and display of directory contents on local host
+ '''
+ def __init__(self, path, batchno=1, batchsize=300, timeout=20,
quiet=False):
+ self.path = path
+ self.st = None
+ self.timeout = timeout
+ self.batchno = batchno
+ self.batchsize = batchsize
+ self.quiet = quiet
+
+ def run(self, quiet=False):
'''
- code = """
-def executor():
- de = DirExaminer('%s', 'localhost', %d, %d, %d)
- de.run()
-""" % (self.path, self.batchno, self.batchsize, self.timeout)
- return code
+ do all the work
+
+ note that 'quiet' applies only to remotely
+ run, and the same is true for returning the contents.
+ maybe we want to fix that
+ '''
+
+ print ('WARNING: trying to get directory contents')
+ dcont = DirContents(self.path, self.batchno, self.batchsize, False)
+ result = dcont.get_contents()
+ if result != 'ok':
+ print ('WARNING: failed to get directory contents'
+ 'for <%s> (%s)'
+ % (self.path, result))
+ else:
+ dcont.get_batch_entryinfo()
+ output = dcont.show_batch()
+ return output
diff --git a/dataretention/retention/retentionaudit.py
b/dataretention/retention/retentionaudit.py
index 8c7aa9b..c3a5ae3 100644
--- a/dataretention/retention/retentionaudit.py
+++ b/dataretention/retention/retentionaudit.py
@@ -7,7 +7,7 @@
from retention.localfileaudit import LocalFilesAuditor
from retention.locallogaudit import LocalLogsAuditor
from retention.localhomeaudit import LocalHomesAuditor
-from retention.examiner import LocalFileExaminer
+from retention.examiner import LocalFileExaminer, LocalDirExaminer
log = logging.getLogger(__name__)
@@ -50,3 +50,11 @@
timeout, quiet)
result = fexaminer.run()
return result
+
+def examine_dir(path, batchno, batchsize,
+ timeout, quiet=False):
+ dexaminer = LocalDirExaminer(path, batchno,
+ batchsize, timeout, quiet)
+ result = dexaminer.run()
+ return result
+
--
To view, visit https://gerrit.wikimedia.org/r/233460
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Idab4a867e4bac7e6e8e433ea2497a04c34edc34d
Gerrit-PatchSet: 2
Gerrit-Project: operations/software
Gerrit-Branch: master
Gerrit-Owner: ArielGlenn <[email protected]>
Gerrit-Reviewer: ArielGlenn <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
