ArielGlenn has uploaded a new change for review. https://gerrit.wikimedia.org/r/120764
Change subject: wikiqueries: pep8 and fixup of usage message ...................................................................... wikiqueries: pep8 and fixup of usage message Change-Id: Ic086433de10657b49ef90befed12f11493466a17 --- M xmldumps-backup/wikiqueries/wikiqueries.py 1 file changed, 146 insertions(+), 88 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/dumps refs/changes/64/120764/1 diff --git a/xmldumps-backup/wikiqueries/wikiqueries.py b/xmldumps-backup/wikiqueries/wikiqueries.py index 520c7d8..8d12983 100644 --- a/xmldumps-backup/wikiqueries/wikiqueries.py +++ b/xmldumps-backup/wikiqueries/wikiqueries.py @@ -16,6 +16,7 @@ import traceback import shutil + class ContentFile(object): def __init__(self, config, date, wikiName): self._config = config @@ -28,25 +29,28 @@ return "content.txt" def getPath(self): - return os.path.join(self.queryDir.getQueryDir(self.wikiName, self.date),self.getFileName()) + return os.path.join(self.queryDir.getQueryDir( + self.wikiName, self.date), self.getFileName()) + class OutputFile(ContentFile): def __init__(self, config, date, wikiName, fileNameFormat): - super( OutputFile, self ).__init__(config, date, wikiName) + super(OutputFile, self).__init__(config, date, wikiName) self.fileNameFormat = fileNameFormat def getFileName(self): return fileNameFormat.format(w=self.wikiName, d=self.date) + class Config(WikiDump.Config): def __init__(self, configFile=False): self.projectName = False home = os.path.dirname(sys.argv[0]) - if (not configFile): + if not configFile: configFile = "wikiqueries.conf" self.files = [ - os.path.join(home,configFile), + os.path.join(home, configFile), "/etc/wikqueries.conf", os.path.join(os.getenv("HOME"), ".wikiqueries.conf")] defaults = { @@ -56,7 +60,7 @@ "closedwikislist": "", #"output": { "wikiqueriesdir": "/wikiqueries", - "temp":"/wikiqueries/temp", + "temp": "/wikiqueries/temp", "fileperms": "0640", #"database": { # these are now set in getDbUserAndPassword() if needed @@ -77,27 +81,34 @@ print "The mandatory configuration section 'wiki' was not defined." raise ConfigParser.NoSectionError('wiki') - if not self.conf.has_option("wiki","mediawiki"): - print "The mandatory setting 'mediawiki' in the section 'wiki' was not defined." - raise ConfigParser.NoOptionError('wiki','mediawiki') + if not self.conf.has_option("wiki", "mediawiki"): + print ("The mandatory setting 'mediawiki' " + "in the section 'wiki' was not defined.") + raise ConfigParser.NoOptionError('wiki', 'mediawiki') self.parseConfFile() - self.getDbUserAndPassword() # get from MW adminsettings file if not set in conf file + # get from MW adminsettings file if not set in conf file + self.getDbUserAndPassword() def parseConfFile(self): self.mediawiki = self.conf.get("wiki", "mediawiki") - self.allWikisList = MiscUtils.dbList(self.conf.get("wiki", "allwikislist")) - self.privateWikisList = MiscUtils.dbList(self.conf.get("wiki", "privatewikislist")) - self.closedWikisList = MiscUtils.dbList(self.conf.get("wiki", "closedwikislist")) + self.allWikisList = MiscUtils.dbList(self.conf.get( + "wiki", "allwikislist")) + self.privateWikisList = MiscUtils.dbList(self.conf.get( + "wiki", "privatewikislist")) + self.closedWikisList = MiscUtils.dbList(self.conf.get( + "wiki", "closedwikislist")) if not self.conf.has_section('output'): self.conf.add_section('output') self.wikiQueriesDir = self.conf.get("output", "wikiqueriesdir") self.tempDir = self.conf.get("output", "temp") self.fileperms = self.conf.get("output", "fileperms") - self.fileperms = int(self.fileperms,0) + self.fileperms = int(self.fileperms, 0) - self.wikiDir = self.mediawiki # the parent class methods want this + # the parent class methods want this + self.wikiDir = self.mediawiki + self.dbUser = None self.dbPassword = None if not self.conf.has_section('database'): @@ -106,7 +117,8 @@ self.dbUser = self.conf.get("database", "user") if self.conf.has_option('database', 'password'): self.dbPassword = self.conf.get("database", "password") - self.getDbUserAndPassword() # get from MW adminsettings file if not set in conf file + # get from MW adminsettings file if not set in conf file + self.getDbUserAndPassword() if not self.conf.has_section('tools'): self.conf.add_section('tools') @@ -114,29 +126,33 @@ self.gzip = self.conf.get("tools", "gzip") self.bzip2 = self.conf.get("tools", "bzip2") self.mysql = self.conf.get("tools", "mysql") - self.multiversion = self.conf.get("tools","multiversion") + self.multiversion = self.conf.get("tools", "multiversion") if not self.conf.has_section('query'): self.conf.add_section('query') - self.queryFile = self.conf.get("query","queryfile") + self.queryFile = self.conf.get("query", "queryfile") + class MultiVersion(object): def MWScriptAsString(config, maintenanceScript): - return(" ".join(MultiVersion.MWScriptAsArray(config, maintenanceScript))) + return(" ".join(MultiVersion.MWScriptAsArray(config, + maintenanceScript))) def MWScriptAsArray(config, maintenanceScript): if config.multiversion != "": if exists(config.multiversion): - return [ config.multiversion, maintenanceScript ] - return [ "%s/maintenance/%s" % (config.mediawiki, maintenanceScript) ] + return [config.multiversion, maintenanceScript] + return ["%s/maintenance/%s" % (config.mediawiki, + maintenanceScript)] MWScriptAsString = staticmethod(MWScriptAsString) MWScriptAsArray = staticmethod(MWScriptAsArray) + class MiscUtils(object): def dbList(filename): """Read database list from a file""" - if (not filename): + if not filename: return [] infile = open(filename) dbs = [] @@ -153,7 +169,8 @@ if isinstance(param, basestring): return "'" + param.replace("'", "'\\''") + "'" elif param is None: - # A blank string might actually be needed; None means we can leave it out + # A blank string might actually be needed; None means + # we can leave it out return "" else: return tuple([MiscUtils.shellEscape(x) for x in param]) @@ -173,57 +190,67 @@ today = staticmethod(today) readFile = staticmethod(readFile) + class RunSimpleCommand(object): - def runWithOutput(command, maxtries = 3, shell=False, verbose = False): + def runWithOutput(command, maxtries=3, shell=False, verbose=False): """Run a command and return the output as a string. Raises WikiQueriesError on non-zero return code.""" - if type(command).__name__=='list': - commandString = " ".join([ "'" + c + "'" for c in command ]) + if type(command).__name__ == 'list': + commandString = " ".join(["'" + c + "'" for c in command]) else: commandString = command if verbose: print "command to be run: ", commandString success = False tries = 0 - while (not success and tries < maxtries): - proc = Popen(command, shell = shell, stdout = PIPE, stderr = PIPE) + while not success and tries < maxtries: + proc = Popen(command, shell=shell, stdout=PIPE, stderr=PIPE) output, error = proc.communicate() if not proc.returncode: success = True tries = tries + 1 if not success: if proc: - raise WikiQueriesError("command '" + commandString + ( "' failed with return code %s " % proc.returncode ) + " and error '" + error + "'") + raise WikiQueriesError("command '" + commandString + + ("' failed with return code %s " + % proc.returncode) + + " and error '" + error + "'") else: - raise WikiQueriesError("command '" + commandString + ( "' failed" ) + " and error '" + error + "'") + raise WikiQueriesError("command '" + commandString + + ("' failed") + " and error '" + + error + "'") return output - def runWithNoOutput(command, maxtries = 3, shell=False, verbose=False): + def runWithNoOutput(command, maxtries=3, shell=False, verbose=False): """Run a command, expecting no output. Raises WikiQueriesError on non-zero return code.""" - if type(command).__name__=='list': - commandString = " ".join([ "'" + c + "'" for c in command ]) + if type(command).__name__ == 'list': + commandString = " ".join(["'" + c + "'" for c in command]) else: commandString = command if verbose: print "command to be run with no output: ", commandString success = False tries = 0 - while ((not success) and tries < maxtries): - proc = Popen(command, shell = shell, stderr = PIPE) + while (not success) and tries < maxtries: + proc = Popen(command, shell=shell, stderr=PIPE) # output will be None, we can ignore it output, error = proc.communicate() if not proc.returncode: success = True tries = tries + 1 if not success: - raise WikiQueriesError("command '" + commandString + ( "' failed with return code %s " % proc.returncode ) + " and error '" + error + "'") + raise WikiQueriesError("command '" + commandString + + ("' failed with return code %s " + % proc.returncode) + " and error '" + + error + "'") return success runWithOutput = staticmethod(runWithOutput) runWithNoOutput = staticmethod(runWithNoOutput) + class DBServer(object): def __init__(self, config, wikiName): @@ -232,27 +259,33 @@ self.dbServer = self.defaultServer() def defaultServer(self): - if (not exists( self.config.php ) ): + if not exists(self.config.php): raise BackupError("php command %s not found" % self.config.php) - commandList = MultiVersion.MWScriptAsArray(self.config, "getSlaveServer.php") - command = [ self.config.php, "-q" ] + commandList = MultiVersion.MWScriptAsArray(self.config, + "getSlaveServer.php") + command = [self.config.php, "-q"] command.extend(commandList) - command.extend( [ "--wiki=%s" % self.wikiName, "--group=dump" ]) + command.extend(["--wiki=%s" % self.wikiName, "--group=dump"]) return RunSimpleCommand.runWithOutput(command, shell=False).rstrip() def buildSqlCommand(self, query, outFile): - """Put together a command to execute an sql query to the server for this DB.""" - if (not exists( self.config.mysql ) ): + """Put together a command to execute an sql query + to the server for this DB.""" + if not exists(self.config.mysql): raise BackupError("mysql command %s not found" % self.config.mysql) - command = "/bin/echo '%s' | %s -h %s -u %s " % ( query, self.config.mysql, self.dbServer, self.config.dbUser ) + command = ("/bin/echo '%s' | %s -h %s -u %s " + % (query, self.config.mysql, self.dbServer, + self.config.dbUser)) if self.config.dbPassword != "": command = command + "-p" + self.config.dbPassword command = command + " -r --silent " + self.wikiName - command = command + "| %s > %s" % ( self.config.gzip, outFile ) + command = command + "| %s > %s" % (self.config.gzip, outFile) return command + class WikiQueriesError(Exception): pass + class QueryDir(object): def __init__(self, config): @@ -261,8 +294,10 @@ def getQueryDir(self, wiki, date): return self._config.wikiQueriesDir.format(w=wiki, d=date) + class WikiQuery(object): - def __init__(self,config, query, wikiName, fileNameFormat, date, overwrite, dryrun, verbose): + def __init__(self, config, query, wikiName, fileNameFormat, date, + overwrite, dryrun, verbose): self._config = config self.wikiName = wikiName self.query = query @@ -279,35 +314,43 @@ def doOneWiki(self): """returns true on success""" - if self.wikiName not in self._config.privateWikisList and self.wikiName not in self._config.closedWikisList: + if (self.wikiName not in self._config.privateWikisList and + self.wikiName not in self._config.closedWikisList): if not exists(self.queryDir.getQueryDir(self.wikiName, self.date)): - os.makedirs(self.queryDir.getQueryDir(self.wikiName, self.date)) + os.makedirs(self.queryDir.getQueryDir(self.wikiName, + self.date)) try: - if (self.verbose): - print "Doing run for wiki: ",self.wikiName + if self.verbose: + print "Doing run for wiki: ", self.wikiName if not dryrun: if not self.runWikiQuery(): return False except: - if (self.verbose): + if self.verbose: traceback.print_exc(file=sys.stdout) return False - if (self.verbose): + if self.verbose: print "Success! Wiki", self.wikiName, "query complete." return True def runWikiQuery(self): - outFile = OutputFile(self._config, self.date, self.wikiName, self.fileNameFormat) + outFile = OutputFile(self._config, self.date, self.wikiName, + self.fileNameFormat) if not self.overwrite and exists(outFile.getPath()): # don't overwrite existing file, just return a happy value if self.verbose: - print "Skipping wiki %s, file exists already" % self.wikiName + print ("Skipping wiki %s, file %s exists already" + % (self.wikiName, outFile.getPath())) return True db = DBServer(self._config, self.wikiName) - return RunSimpleCommand.runWithNoOutput(db.buildSqlCommand(self.query, outFile.getPath()), maxtries=1, shell=True, verbose=self.verbose) + return RunSimpleCommand.runWithNoOutput(db.buildSqlCommand( + self.query, outFile.getPath()), maxtries=1, shell=True, + verbose=self.verbose) + class WikiQueryLoop(object): - def __init__(self, config, query, fileNameFormat, date, overwrite, dryrun, verbose): + def __init__(self, config, query, fileNameFormat, date, overwrite, + dryrun, verbose): self._config = config self.query = query self.date = date @@ -322,48 +365,58 @@ def doRunOnAllWikis(self): failures = 0 for w in self.wikisToDo[:]: - query = WikiQuery(self._config, self.query, w, self.fileNameFormat, self.date, self.overwrite, self.dryrun, self.verbose) + query = WikiQuery(self._config, self.query, w, + self.fileNameFormat, self.date, + self.overwrite, self.dryrun, self.verbose) if query.doOneWiki(): self.wikisToDo.remove(w) def doAllWikisTilDone(self, numFails): - """Run through all wikis, retrying up to numFails + """Run through all wikis, retrying up to numFails times in case of error""" fails = 0 while 1: self.doRunOnAllWikis() if not len(self.wikisToDo): break - fails = fails + 1 + fails = fails + 1 if fails > numFails: raise WikiQueriesError("Too many failures, giving up") # wait 5 minutes and try another loop time.sleep(300) -def usage(message = None): + +def usage(message=None): if message: - print message - print "Usage: python wikiqueries.py [options] [wikidbname]" - print "Options: --configfile, --date, --dryrun, --filenameformat, --outdir, --query, --retries, --verbose" - print "--configfile: Specify config file to read." - print " Default: wikiqueries.conf" - print "--date: date that will appear in filename and/or dirname as specified. Format: YYYYMMDD" - print " If not specified, today's date will be used." - print "--dryrun: Don't actually run anything but print the commands that would be run." - print "--filenameformat: Format string for the name of each file, with {w} for wikiname and optional" - print " {d} for date." - print " Default: {w}-{d}-wikiquery.gz" - print "--outdir: Put output files for all projects in this directory; it will be created if" - print " it does not exist. Accepts '{W}' and '{d}' for substituting wiki and date into the name." - print " Default: the value given for 'querydir' in the config file" - print "--nooverwrite: Do not overwrite existing file of the same name, skip run for the specific wiki" - print "--query: MySQL query to run on each project." - print " Default: the contents of the file specified by 'queryfile' in the config file" - print "--retries: Number of times to try running the query on all wikis in case of error, before giving up." - print " Default: 3" - print "--verbose: Print various informative messages." - print "wikiname: Run the query only for the specific wiki." - sys.exit(1) + sys.stderr.write(message + "\n") + usage_message = """Usage: python wikiqueries.py [options] [wikidbname] +Options: + +--configfile: Specify config file to read + Default: wikiqueries.conf +--date: date that will appear in filename and/or dirname as specified + Format: YYYYMMDD If not specified, today's date will be used +--dryrun: Don't execute commands, show the commands that would be run +--filenameformat: Format string for the name of each file, with {w} for + wikiname and optional {d} for date + Default: {w}-{d}-wikiquery.gz +--outdir: Put output files for all projects in this directory; it will + be created if it does not exist. Accepts '{W}' and '{d}' for + substituting wiki and date into the name. + Default: the value given for 'querydir' in the config file +--nooverwrite: Do not overwrite existing file of the same name, skip run for + the specific wiki +--query: MySQL query to run on each project. + Default: the contents of the file specified by 'queryfile' in + the config file +--retries: Number of times to try running the query on all wikis in case + of error, before giving up + Default: 3 +--verbose: Print various informative messages +wikiname: Run the query only for the specific wiki +""" + sys.stderr.write(usage_message) + sys.exit(1) if __name__ == "__main__": configFile = False @@ -377,8 +430,10 @@ verbose = False fileNameFormat = "{w}-{d}-wikiquery.gz" try: - (options, remainder) = getopt.gnu_getopt(sys.argv[1:], "", - [ 'configfile=', "date=", 'filenameformat=', "outdir=", "query=", "retries=", 'dryrun', "nooverwrite", 'verbose' ]) + (options, remainder) = getopt.gnu_getopt( + sys.argv[1:], "", ['configfile=', "date=", 'filenameformat=', + "outdir=", "query=", "retries=", 'dryrun', + "nooverwrite", 'verbose']) except: usage("Unknown option specified") @@ -403,11 +458,12 @@ retries = int(val) elif opt == "--verbose": verbose = True - - if date and not re.match("^20[0-9]{6}$", date): - usage("Date must be in the format YYYYMMDD (four digit year, two digit month, two digit date)") - if (configFile): + if date and not re.match("^20[0-9]{6}$", date): + usage("Date must be in the format YYYYMMDD" + " (four digit year, two digit month, two digit date)") + + if configFile: config = Config(configFile) else: config = Config() @@ -416,8 +472,10 @@ config.wikiQueriesDir = outputDir if len(remainder) > 0: - query = WikiQuery(config, query, remainder[0], fileNameFormat, date, overwrite, dryrun, verbose) + query = WikiQuery(config, query, remainder[0], fileNameFormat, + date, overwrite, dryrun, verbose) query.doOneWiki() else: - queries = WikiQueryLoop(config, query, fileNameFormat, date, overwrite, dryrun, verbose) + queries = WikiQueryLoop(config, query, fileNameFormat, date, + overwrite, dryrun, verbose) queries.doAllWikisTilDone(retries) -- To view, visit https://gerrit.wikimedia.org/r/120764 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ic086433de10657b49ef90befed12f11493466a17 Gerrit-PatchSet: 1 Gerrit-Project: operations/dumps Gerrit-Branch: ariel Gerrit-Owner: ArielGlenn <ar...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits