ArielGlenn has submitted this change and it was merged. Change subject: pep8: E302 expected 2 blank lines, found 1 ......................................................................
pep8: E302 expected 2 blank lines, found 1 Change-Id: I0b60eccc88c32e96637abeb84c82abf099ed9a44 --- M WikiDump.py M monitor.py M worker.py 3 files changed, 32 insertions(+), 0 deletions(-) Approvals: ArielGlenn: Verified; Looks good to me, approved jenkins-bot: Checked diff --git a/WikiDump.py b/WikiDump.py index cca4160..49699b4 100644 --- a/WikiDump.py +++ b/WikiDump.py @@ -8,13 +8,16 @@ import threading import time + def fileAge(filename): return time.time() - os.stat(filename).st_mtime + def atomicCreate(filename, mode='w'): """Create a file, aborting if it already exists...""" fd = os.open(filename, os.O_EXCL + os.O_CREAT + os.O_WRONLY) return os.fdopen(fd, mode) + def shellEscape(param): """Escape a string parameter, or set of strings, for the shell.""" @@ -26,10 +29,12 @@ else: return tuple([shellEscape(x) for x in param]) + def prettySize(size): """Return a string with an attractively formatted file size.""" quanta = ("%d bytes", "%d KB", "%0.1f MB", "%0.1f GB", "%0.1f TB") return _prettySize(size, quanta) + def _prettySize(size, quanta): if size < 1024 or len(quanta) == 1: @@ -37,15 +42,19 @@ else: return _prettySize(size / 1024.0, quanta[1:]) + def today(): return time.strftime("%Y%m%d", time.gmtime()) + def prettyTime(): return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()) + def prettyDate(key): "Prettify a MediaWiki date key" return "-".join((key[0:4], key[4:6], key[6:8])) + def dumpFile(filename, text): """Dump a string to a file, as atomically as possible, via a temporary file in the same directory.""" @@ -61,11 +70,13 @@ # Of course nothing else will work on Windows. ;) os.rename(tempFilename, filename) + def readFile(filename): file = open(filename, "r") text = file.read() file.close() return text + def dbList(filename): """Read database list from a file""" @@ -78,6 +89,7 @@ infile.close() dbs.sort() return dbs + class Config(object): def __init__(self): @@ -353,6 +365,7 @@ def lockAge(self): return fileAge(self.lockFile()) + class LockWatchdog(threading.Thread): """Touch the given file every 10 seconds until asked to stop.""" @@ -389,6 +402,7 @@ """Run me inside...""" os.utime(self.lockfile, None) + def cleanup(): """Call cleanup handlers for any background threads...""" for watchdog in LockWatchdog.threads: diff --git a/monitor.py b/monitor.py index 5c229a4..be2e429 100644 --- a/monitor.py +++ b/monitor.py @@ -5,6 +5,7 @@ config = WikiDump.Config() + def generateIndex(): running = False states = [] @@ -34,6 +35,7 @@ "status": status, "items": "\n".join(states)} + def updateIndex(): outputFileName = os.path.join(config.publicDir, config.index) WikiDump.dumpFile(outputFileName, generateIndex()) diff --git a/worker.py b/worker.py index cafc246..6092f8a 100644 --- a/worker.py +++ b/worker.py @@ -12,6 +12,7 @@ from os.path import dirname, exists, getsize, join, realpath from WikiDump import prettyTime, prettySize, shellEscape + def splitPath(path): # For some reason, os.path.split only does one level. parts = [] @@ -24,6 +25,7 @@ (path, file) = os.path.split(path) return parts + def relativePath(path, base): """Return a relative path to 'path' from the directory 'base'.""" path = splitPath(path) @@ -34,6 +36,7 @@ for prefix in base: path.insert(0, "..") return os.path.join(*path) + def md5File(filename): summer = md5.new() @@ -46,14 +49,18 @@ infile.close() return summer.hexdigest() + def md5FileLine(filename): return "%s %s\n" % (md5File(filename), os.path.basename(filename)) + def xmlEscape(text): return text.replace("&", "&").replace("<", "<").replace(">", ">") + class BackupError(Exception): pass + class Runner(object): @@ -559,6 +566,7 @@ rssPath = self.latestPath(file + "-rss.xml") WikiDump.dumpFile(rssPath, rssText) + class Dump(object): def __init__(self, desc): self._desc = desc @@ -608,6 +616,7 @@ def matchCheckpoint(self, checkpoint): return checkpoint == self.__class__.__name__ + class PublicTable(Dump): """Dump of a table using MySQL's mysqldump utility.""" @@ -629,6 +638,7 @@ def matchCheckpoint(self, checkpoint): return checkpoint == self.__class__.__name__ + "." + self._table + class PrivateTable(PublicTable): """Hidden table dumps for private data.""" @@ -693,6 +703,7 @@ articles)) runner.runCommand(command, callback=self.progressCallback) + class XmlLogging(Dump): """ Create a logging dump of all page activity """ @@ -725,6 +736,7 @@ runner.dbServer, logging)) runner.runCommand(command, callback=self.progressCallback) + class XmlDump(Dump): """Primary XML dumps, one section at a time.""" @@ -840,6 +852,7 @@ def matchCheckpoint(self, checkpoint): return checkpoint == self.__class__.__name__ + "." + self._subset + class BigXmlDump(XmlDump): """XML page dump for something larger, where a 7-Zip compressed copy could save 75% of download time for some users.""" @@ -847,6 +860,7 @@ def buildEta(self, runner): """Tell the dumper script whether to make ETA estimate on page or revision count.""" return "--full" + class XmlRecompressDump(Dump): """Take a .bz2 and recompress it as 7-Zip.""" @@ -892,6 +906,7 @@ def matchCheckpoint(self, checkpoint): return checkpoint == self.__class__.__name__ + "." + self._subset + class AbstractDump(Dump): """XML dump for Yahoo!'s Active Abstracts thingy""" @@ -947,6 +962,7 @@ def listFiles(self, runner): return [self._variantFile(x) for x in self._variants(runner)] + class TitleDump(Dump): """This is used by "wikiproxy", a program to add Wikipedia links to BBC news online""" def run(self, runner): -- To view, visit https://gerrit.wikimedia.org/r/50370 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I0b60eccc88c32e96637abeb84c82abf099ed9a44 Gerrit-PatchSet: 1 Gerrit-Project: operations/dumps Gerrit-Branch: master Gerrit-Owner: Hashar <has...@free.fr> Gerrit-Reviewer: ArielGlenn <ar...@wikimedia.org> Gerrit-Reviewer: jenkins-bot _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits