Hoo man has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/391053 )
Change subject: Use --no-cache for dumping Wikidata entities
......................................................................
Use --no-cache for dumping Wikidata entities
This flag will work once 795350da2 is deployed, but
passing it earlier is fine (MW discards those).
Bug: T180048
Change-Id: Iac6fa3fb055834cebc1f10ba472d63671cb1322a
---
M modules/snapshot/files/cron/dumpwikidatajson.sh
M modules/snapshot/files/cron/dumpwikidatardf.sh
2 files changed, 4 insertions(+), 2 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/53/391053/1
diff --git a/modules/snapshot/files/cron/dumpwikidatajson.sh
b/modules/snapshot/files/cron/dumpwikidatajson.sh
index a53ac8d..49136ab 100644
--- a/modules/snapshot/files/cron/dumpwikidatajson.sh
+++ b/modules/snapshot/files/cron/dumpwikidatajson.sh
@@ -30,7 +30,8 @@
set -o pipefail
errorLog=/var/log/wikidatadump/dumpwikidatajson-$filename-$i.log
# NOTE: We temporary set the shard size differently for
each shard. T177486#3674942.
- php5 $multiversionscript
extensions/Wikidata/extensions/Wikibase/repo/maintenance/dumpJson.php --wiki
wikidatawiki --shard $i --sharding-factor $shards --batch-size `expr $(expr $i
+ 1) \* 500` --snippet 2>> $errorLog | gzip -9 > $tempDir/wikidataJson.$i.gz
+ # Remove --no-cache once this runs on hhvm (or
everything is back on Zend), see T180048.
+ php5 $multiversionscript
extensions/Wikidata/extensions/Wikibase/repo/maintenance/dumpJson.php --wiki
wikidatawiki --shard $i --sharding-factor $shards --batch-size `expr $(expr $i
+ 1) \* 500` --snippet 2 --no-cache >> $errorLog | gzip -9 >
$tempDir/wikidataJson.$i.gz
exitCode=$?
if [ $exitCode -gt 0 ]; then
echo -e "\n\n(`date --iso-8601=minutes`)
Process for shard $i failed with exit code $exitCode" >> $errorLog
diff --git a/modules/snapshot/files/cron/dumpwikidatardf.sh
b/modules/snapshot/files/cron/dumpwikidatardf.sh
index 006c36d..c3f866f 100755
--- a/modules/snapshot/files/cron/dumpwikidatardf.sh
+++ b/modules/snapshot/files/cron/dumpwikidatardf.sh
@@ -56,7 +56,8 @@
(
set -o pipefail
errorLog=/var/log/wikidatadump/dumpwikidata$dumpFormat-$filename-$i.log
- php5 $multiversionscript
extensions/Wikidata/extensions/Wikibase/repo/maintenance/dumpRdf.php --wiki
wikidatawiki --shard $i --sharding-factor $shards --batch-size `expr $shards \*
500` --format $dumpFormat --flavor $dumpFlavor 2>> $errorLog | gzip -9 >
$tempDir/wikidata$dumpFormat-$dumpName.$i.gz
+ # Remove --no-cache once this runs on hhvm (or
everything is back on Zend), see T180048.
+ php5 $multiversionscript
extensions/Wikidata/extensions/Wikibase/repo/maintenance/dumpRdf.php --wiki
wikidatawiki --shard $i --sharding-factor $shards --batch-size `expr $shards \*
500` --format $dumpFormat --flavor $dumpFlavor --no-cache 2>> $errorLog | gzip
-9 > $tempDir/wikidata$dumpFormat-$dumpName.$i.gz
exitCode=$?
if [ $exitCode -gt 0 ]; then
echo -e "\n\n(`date --iso-8601=minutes`)
Process for shard $i failed with exit code $exitCode" >> $errorLog
--
To view, visit https://gerrit.wikimedia.org/r/391053
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Iac6fa3fb055834cebc1f10ba472d63671cb1322a
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Hoo man <h...@online.de>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
