[MediaWiki-commits] [Gerrit] download only when needed - change (analytics/zero-sms)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/159976 Change subject: download only when needed .. download only when needed Change-Id: Ie36b9be8881f86d02d7d057ebffc847a48f2ab4c --- M scripts/smslogs.py M scripts/weblogs.py 2 files changed, 36 insertions(+), 15 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/analytics/zero-sms refs/changes/76/159976/1 diff --git a/scripts/smslogs.py b/scripts/smslogs.py index 40cadf9..7f01c2e 100644 --- a/scripts/smslogs.py +++ b/scripts/smslogs.py @@ -112,6 +112,7 @@ bucket = cn.get_bucket(self.settings.awsBucket) files = bucket.list(self.settings.awsPrefix) +newDataFound = False for key in files: filename = key.key[len(self.settings.awsPrefix):] @@ -160,8 +161,15 @@ key.get_contents_to_filename(filePath) if fileDate and (not self.settings.lastDownloadTs or self.settings.lastDownloadTs < fileDate): self.settings.lastDownloadTs = fileDate +newDataFound = True -def combineDataFiles(self, sourceFiles): +return newDataFound + +def combineDataFiles(self): + +sourceFiles = os.listdir(self.pathLogs) +# files = itertools.chain([os.path.join('pc', f) for f in os.listdir(os.path.join(self.pathLogs, 'pc'))], +# files) safePrint(u'Combining files into %s' % self.combinedFilePath) if self.processIfAfter: @@ -169,11 +177,11 @@ else: safePrint(u'Processing all files') -appendingDataFile = self.combinedFilePath + '.tmp' +tempFile = self.combinedFilePath + '.tmp' manualLogRe = re.compile(r'^wikipedia_application_\d+\.log\.\d+\.gz:') totalCount = 0 -with io.open(appendingDataFile, 'w', encoding='utf8') as dst: +with io.open(tempFile, 'w', encoding='utf8') as dst: for srcFile in sourceFiles: fileDate = self.getFileDate(srcFile) @@ -216,7 +224,7 @@ if os.path.exists(sortedOutputFile): os.remove(sortedOutputFile) -args = [self.settings.sortCmd, '-u', '-o', sortedOutputFile, appendingDataFile] +args = [self.settings.sortCmd, '-u', '-o', sortedOutputFile, tempFile] originalExists = os.path.exists(self.combinedFilePath) if originalExists: args.append(self.combinedFilePath) @@ -239,7 +247,7 @@ except subprocess.CalledProcessError, ex: raise Exception(u'Error %s running %s\nOutput:\n%s' % (ex.returncode, cmd, ex.output)) -os.remove(appendingDataFile) +os.remove(tempFile) def generateGraphData(self, skipParsing=False): stats = smsgraphs.Stats(self.combinedFilePath, self.pathGraphs, self.statsFilePath, self.settings.partnerMap, @@ -257,13 +265,15 @@ stats.createGraphs() def run(self): +newDataFound = True if self.settings.enableDownload: -self.download() -files = os.listdir(self.pathLogs) -files = itertools.chain([os.path.join('pc', f) for f in os.listdir(os.path.join(self.pathLogs, 'pc'))], -files) -self.combineDataFiles(files) -self.generateGraphData() +newDataFound = self.download() + +if not newDataFound and os.path.isfile(self.combinedFilePath): +safePrint('No new data, we are done') +else: +self.combineDataFiles() +self.generateGraphData() if __name__ == "__main__": diff --git a/scripts/weblogs.py b/scripts/weblogs.py index 4abb5b0..7e5453a 100644 --- a/scripts/weblogs.py +++ b/scripts/weblogs.py @@ -36,6 +36,7 @@ self.urlRe = re.compile(r'^https?://([^/]+)', re.IGNORECASE) self.duplUrlRe = re.compile(r'^(https?://.+)\1', re.IGNORECASE) self.zcmdRe = re.compile(r'zcmd=([-a-z0-9]+)', re.IGNORECASE) +self.combinedFile = os.path.join(self.pathGraphs, 'combined-all.tsv') def defaultSettings(self, suffix): s = super(WebLogProcessor, self).defaultSettings(suffix) @@ -68,6 +69,7 @@ def processLogFiles(self): +newDataFound = False safePrint('Processing log files') statFiles = {} for f in os.listdir(self.pathLogs): @@ -82,6 +84,7 @@ fileDt = m.group(1) fileDt = '-'.join([fileDt[0:4], fileDt[4:6], fileDt[6:8]]) self.processLogFile(logFile, statFile, fileDt) +newDataFound = True # Clean up older stat files (if gz file size has changed) removeFiles = [] @@ -96,6 +99,8 @@ removeFiles.append(statFile) for f in removeFiles: os.remove(f) + +return newDataFound def processLogFile(self, logFile, statFile, fileDt): """ @@ -272
[MediaWiki-commits] [Gerrit] download only when needed - change (analytics/zero-sms)
Yurik has submitted this change and it was merged. Change subject: download only when needed .. download only when needed Change-Id: Ie36b9be8881f86d02d7d057ebffc847a48f2ab4c --- M scripts/smslogs.py M scripts/weblogs.py 2 files changed, 36 insertions(+), 15 deletions(-) Approvals: Yurik: Verified; Looks good to me, approved diff --git a/scripts/smslogs.py b/scripts/smslogs.py index 40cadf9..7f01c2e 100644 --- a/scripts/smslogs.py +++ b/scripts/smslogs.py @@ -112,6 +112,7 @@ bucket = cn.get_bucket(self.settings.awsBucket) files = bucket.list(self.settings.awsPrefix) +newDataFound = False for key in files: filename = key.key[len(self.settings.awsPrefix):] @@ -160,8 +161,15 @@ key.get_contents_to_filename(filePath) if fileDate and (not self.settings.lastDownloadTs or self.settings.lastDownloadTs < fileDate): self.settings.lastDownloadTs = fileDate +newDataFound = True -def combineDataFiles(self, sourceFiles): +return newDataFound + +def combineDataFiles(self): + +sourceFiles = os.listdir(self.pathLogs) +# files = itertools.chain([os.path.join('pc', f) for f in os.listdir(os.path.join(self.pathLogs, 'pc'))], +# files) safePrint(u'Combining files into %s' % self.combinedFilePath) if self.processIfAfter: @@ -169,11 +177,11 @@ else: safePrint(u'Processing all files') -appendingDataFile = self.combinedFilePath + '.tmp' +tempFile = self.combinedFilePath + '.tmp' manualLogRe = re.compile(r'^wikipedia_application_\d+\.log\.\d+\.gz:') totalCount = 0 -with io.open(appendingDataFile, 'w', encoding='utf8') as dst: +with io.open(tempFile, 'w', encoding='utf8') as dst: for srcFile in sourceFiles: fileDate = self.getFileDate(srcFile) @@ -216,7 +224,7 @@ if os.path.exists(sortedOutputFile): os.remove(sortedOutputFile) -args = [self.settings.sortCmd, '-u', '-o', sortedOutputFile, appendingDataFile] +args = [self.settings.sortCmd, '-u', '-o', sortedOutputFile, tempFile] originalExists = os.path.exists(self.combinedFilePath) if originalExists: args.append(self.combinedFilePath) @@ -239,7 +247,7 @@ except subprocess.CalledProcessError, ex: raise Exception(u'Error %s running %s\nOutput:\n%s' % (ex.returncode, cmd, ex.output)) -os.remove(appendingDataFile) +os.remove(tempFile) def generateGraphData(self, skipParsing=False): stats = smsgraphs.Stats(self.combinedFilePath, self.pathGraphs, self.statsFilePath, self.settings.partnerMap, @@ -257,13 +265,15 @@ stats.createGraphs() def run(self): +newDataFound = True if self.settings.enableDownload: -self.download() -files = os.listdir(self.pathLogs) -files = itertools.chain([os.path.join('pc', f) for f in os.listdir(os.path.join(self.pathLogs, 'pc'))], -files) -self.combineDataFiles(files) -self.generateGraphData() +newDataFound = self.download() + +if not newDataFound and os.path.isfile(self.combinedFilePath): +safePrint('No new data, we are done') +else: +self.combineDataFiles() +self.generateGraphData() if __name__ == "__main__": diff --git a/scripts/weblogs.py b/scripts/weblogs.py index 4abb5b0..7e5453a 100644 --- a/scripts/weblogs.py +++ b/scripts/weblogs.py @@ -36,6 +36,7 @@ self.urlRe = re.compile(r'^https?://([^/]+)', re.IGNORECASE) self.duplUrlRe = re.compile(r'^(https?://.+)\1', re.IGNORECASE) self.zcmdRe = re.compile(r'zcmd=([-a-z0-9]+)', re.IGNORECASE) +self.combinedFile = os.path.join(self.pathGraphs, 'combined-all.tsv') def defaultSettings(self, suffix): s = super(WebLogProcessor, self).defaultSettings(suffix) @@ -68,6 +69,7 @@ def processLogFiles(self): +newDataFound = False safePrint('Processing log files') statFiles = {} for f in os.listdir(self.pathLogs): @@ -82,6 +84,7 @@ fileDt = m.group(1) fileDt = '-'.join([fileDt[0:4], fileDt[4:6], fileDt[6:8]]) self.processLogFile(logFile, statFile, fileDt) +newDataFound = True # Clean up older stat files (if gz file size has changed) removeFiles = [] @@ -96,6 +99,8 @@ removeFiles.append(statFile) for f in removeFiles: os.remove(f) + +return newDataFound def processLogFile(self, logFile, statFile, fileDt): """ @@ -272,7 +277,10 @@ stats[key] += int(count)