ArielGlenn has uploaded a new change for review.
https://gerrit.wikimedia.org/r/65689
Change subject: include dump of (one of the) wikidata tables
......................................................................
include dump of (one of the) wikidata tables
Change-Id: I9126664d4ebbfabaf60a849b8d9b7a2735d0cff7
---
M xmldumps-backup/README.config
M xmldumps-backup/WikiDump.py
M xmldumps-backup/wikidump.conf.sample
M xmldumps-backup/worker.py
4 files changed, 14 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/dumps
refs/changes/89/65689/1
diff --git a/xmldumps-backup/README.config b/xmldumps-backup/README.config
index e3613ce..41a8699 100644
--- a/xmldumps-backup/README.config
+++ b/xmldumps-backup/README.config
@@ -59,6 +59,9 @@
flaggedrevslist -- File with list of databases which have flagged revisions
enabled. (Really, we should be able to determine this
another way instead of keeping a separate list, right?)
+wikidatalist -- File with list of databases which act as a wikibase
+ repo. For Wikimedia projects this currently consists
+ of the project 'wikidata'.
biglist -- File with list of large wikis for which no history dumps are
generated because they are too huge. (This must be an old
deprecated option; these days we do not care how big they
diff --git a/xmldumps-backup/WikiDump.py b/xmldumps-backup/WikiDump.py
index ad22b7d..0c48cf0 100644
--- a/xmldumps-backup/WikiDump.py
+++ b/xmldumps-backup/WikiDump.py
@@ -175,6 +175,7 @@
"dblist": "",
"privatelist": "",
"flaggedrevslist": "",
+ "wikidatalist": "",
# "dir": "",
"forcenormal": "0",
"halt": "0",
@@ -258,6 +259,7 @@
self.skipDbList = MiscUtils.dbList(self.conf.get("wiki",
"skipdblist"))
self.privateList = MiscUtils.dbList(self.conf.get("wiki",
"privatelist"))
self.flaggedRevsList = MiscUtils.dbList(self.conf.get("wiki",
"flaggedrevslist"))
+ self.wikidataList = MiscUtils.dbList(self.conf.get("wiki",
"wikidatalist"))
self.wikiDir = self.conf.get("wiki", "dir")
self.forceNormal = self.conf.getint("wiki", "forcenormal")
self.halt = self.conf.getint("wiki", "halt")
@@ -423,6 +425,9 @@
def hasFlaggedRevs(self):
return self.dbName in self.config.flaggedRevsList
+
+ def hasWikidata(self):
+ return self.dbName in self.config.wikidataList
def isLocked(self):
return os.path.exists(self.lockFile())
diff --git a/xmldumps-backup/wikidump.conf.sample
b/xmldumps-backup/wikidump.conf.sample
index 5feed3b..57de2e0 100644
--- a/xmldumps-backup/wikidump.conf.sample
+++ b/xmldumps-backup/wikidump.conf.sample
@@ -5,6 +5,7 @@
skipdblist=/home/ariel/src/mediawiki/testing/backup/skip.dblist
privatelist=/home/ariel/src/mediawiki/testing/backup/private.dblist
flaggedrevslist=/home/ariel/src/mediawiki/testing/backup/flagged.dblist
+wikidatalist=/home/ariel/src/mediawiki/testing/backup/wikidata.dblist
biglist=/home/ariel/src/mediawiki/testing/backup/big.dblist
dir=/home/ariel/src/mediawiki/1.16wmf4/phase3
forcenormal=1
diff --git a/xmldumps-backup/worker.py b/xmldumps-backup/worker.py
index 08c3816..fc6b015 100644
--- a/xmldumps-backup/worker.py
+++ b/xmldumps-backup/worker.py
@@ -557,6 +557,7 @@
def __init__(self, wiki, prefetch, spawn, chunkToDo, checkpointFile,
singleJob, chunkInfo, pageIDRange, runInfoFile, dumpDir):
self.wiki = wiki
self._hasFlaggedRevs = self.wiki.hasFlaggedRevs()
+ self._hasWikidata = self.wiki.hasWikidata()
self._prefetch = prefetch
self._spawn = spawn
self.chunkInfo = chunkInfo
@@ -663,6 +664,10 @@
self.dumpItems.append(
PublicTable( "flaggedrevs",
"flaggedrevstable","This contains a row for each flagged revision, containing
who flagged it, when it was flagged, reviewer comments, the flag values, and
the quality tier those flags fall under." ))
+ if self._hasWikidata:
+ self.dumpItems.append(
+ PublicTable( "wb_items_per_site",
"wbitemspersitetable","For each Wikidata item, this contains rows with the
corresnponding page name on a given wiki project." ))
+
self.dumpItems.append(
BigXmlDump("meta-history",
"metahistorybz2dump",
--
To view, visit https://gerrit.wikimedia.org/r/65689
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I9126664d4ebbfabaf60a849b8d9b7a2735d0cff7
Gerrit-PatchSet: 1
Gerrit-Project: operations/dumps
Gerrit-Branch: ariel
Gerrit-Owner: ArielGlenn <ar...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
