ArielGlenn has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/342311 )
Change subject: have dump monitor collect current run json files and produce
index.json
......................................................................
have dump monitor collect current run json files and produce index.json
Bug: T147177
Change-Id: I66b4c5ba07016214c8e2140a8ebd6629755de663
---
M xmldumps-backup/dumps/runstatusapi.py
M xmldumps-backup/monitor.py
2 files changed, 54 insertions(+), 0 deletions(-)
Approvals:
ArielGlenn: Looks good to me, approved
jenkins-bot: Verified
diff --git a/xmldumps-backup/dumps/runstatusapi.py
b/xmldumps-backup/dumps/runstatusapi.py
index a82c3b6..192c4d4 100644
--- a/xmldumps-backup/dumps/runstatusapi.py
+++ b/xmldumps-backup/dumps/runstatusapi.py
@@ -29,6 +29,25 @@
# might add more someday, but not today
known_formats = ["json"]
+ @staticmethod
+ def get_wiki_info(wiki, fmt="json"):
+ """
+ read and return the contents of the json status file
+ for the wiki
+ """
+ if fmt not in StatusAPI.known_formats:
+ return {}
+ date = wiki.latest_dump()
+ if date:
+ fname = os.path.join(wiki.public_dir(),
+ date, StatusAPI.FILENAME + "." + fmt)
+ with open(fname, "r") as status_file:
+ contents = status_file.read()
+ status_file.close()
+ return json.loads(contents)
+ else:
+ return {}
+
def __init__(self, wiki, enabled, fileformat="json", error_callback=None,
verbose=False):
self.wiki = wiki
self._enabled = enabled
diff --git a/xmldumps-backup/monitor.py b/xmldumps-backup/monitor.py
index d38b213..60609d7 100644
--- a/xmldumps-backup/monitor.py
+++ b/xmldumps-backup/monitor.py
@@ -4,9 +4,11 @@
from os.path import exists
import sys
import traceback
+import json
from dumps.WikiDump import Wiki, Config, Locker
from dumps.fileutils import FileUtils
from dumps.runnerutils import StatusHtml
+from dumps.runstatusapi import StatusAPI
VERBOSE = False
@@ -64,6 +66,29 @@
"items": "\n".join(states)}
+def generate_json(config):
+ """
+ go through all the latest dump dirs, collect up all the json
+ contents from the dumpstatusapi file, and shovel them into
+ one ginormous json object and scribble that out. heh.
+ """
+ running = False
+ json_out = {"wikis": {}}
+
+ dbs = config.db_list
+
+ for db_name in dbs:
+ try:
+ wiki = Wiki(config, db_name)
+ json_out["wikis"][wiki.db_name] = StatusAPI.get_wiki_info(wiki)
+ except Exception:
+ # if there's a problem with one wiki at least
+ # let's show the rest
+ if VERBOSE:
+ traceback.print_exc(file=sys.stdout)
+ return json_out
+
+
def update_index(config):
output_fname = os.path.join(config.public_dir, config.index)
output_fname_sorted_by_db = add_to_filename(os.path.join(
@@ -83,6 +108,15 @@
os.rename(temp_fname, output_fname_sorted_by_db)
+def update_json(config):
+ output_fname = os.path.join(config.public_dir, "index.json")
+ temp_fname = output_fname + ".tmp"
+ filehdl = open(temp_fname, "wt")
+ filehdl.write(json.dumps(generate_json(config)))
+ filehdl.close()
+ os.rename(temp_fname, output_fname)
+
+
def main():
# can specify name of alternate config file
if len(sys.argv) >= 2:
@@ -90,6 +124,7 @@
else:
config = Config()
update_index(config)
+ update_json(config)
if __name__ == "__main__":
--
To view, visit https://gerrit.wikimedia.org/r/342311
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I66b4c5ba07016214c8e2140a8ebd6629755de663
Gerrit-PatchSet: 3
Gerrit-Project: operations/dumps
Gerrit-Branch: master
Gerrit-Owner: ArielGlenn <[email protected]>
Gerrit-Reviewer: ArielGlenn <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
