http://www.mediawiki.org/wiki/Special:Code/MediaWiki/97344
Revision: 97344
Author: reedy
Date: 2011-09-16 22:54:57 +0000 (Fri, 16 Sep 2011)
Log Message:
-----------
Copy a few scripts out of 1.17wmf1 for preservation
Added Paths:
-----------
trunk/extensions/WikimediaMaintenance/jeluf.php
trunk/extensions/WikimediaMaintenance/jobs-loop.sh
trunk/extensions/WikimediaMaintenance/storage/make-all-blobs
trunk/extensions/WikimediaMaintenance/storage/testRctComplete.php
Copied: trunk/extensions/WikimediaMaintenance/jeluf.php (from rev 97332,
branches/wmf/1.17wmf1/maintenance/jeluf.php)
===================================================================
--- trunk/extensions/WikimediaMaintenance/jeluf.php
(rev 0)
+++ trunk/extensions/WikimediaMaintenance/jeluf.php 2011-09-16 22:54:57 UTC
(rev 97344)
@@ -0,0 +1,103 @@
+<?php
+/**
+ * This script starts pending jobs.
+ *
+ * Usage:
+ * --maxjobs <num> (default 10000)
+ * --type <job_cmd>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @ingroup Maintenance
+ */
+
+require_once( dirname(__FILE__) . '/Maintenance.php' );
+
+class RunJobs extends Maintenance {
+ public function __construct() {
+ parent::__construct();
+ $this->mDescription = "Run pending jobs";
+ $this->addOption( 'maxjobs', 'Maximum number of jobs to run',
false, true );
+ $this->addOption( 'type', 'Type of job to run', false, true );
+ $this->addOption( 'procs', 'Number of processes to use', false,
true );
+ }
+
+ public function memoryLimit() {
+ // Don't eat all memory on the machine if we get a bad job.
+ return "150M";
+ }
+
+ public function execute() {
+ global $wgTitle;
+ if ( $this->hasOption( 'procs' ) ) {
+ $procs = intval( $this->getOption('procs') );
+ if ( $procs < 1 || $procs > 1000 ) {
+ $this->error( "Invalid argument to --procs",
true );
+ }
+ $fc = new ForkController( $procs );
+ if ( $fc->start( $procs ) != 'child' ) {
+ exit( 0 );
+ }
+ }
+ $maxJobs = $this->getOption( 'maxjobs', 10000 );
+ $type = $this->getOption( 'type', false );
+ $wgTitle = Title::newFromText( 'RunJobs.php' );
+ $dbw = wfGetDB( DB_MASTER );
+ $n = 0;
+ $conds = '';
+ if ($type !== false)
+ $conds = "job_cmd = " . $dbw->addQuotes($type);
+
+ while ( $dbw->selectField( 'job', 'job_id', $conds,
'runJobs.php' ) ) {
+ $offset=0;
+ for (;;) {
+ $job = ($type == false) ?
+ Job::pop($offset)
+ : Job::pop_type($type);
+
+ if ($job == false)
+ break;
+
+ wfWaitForSlaves( 5 );
+ $t = microtime( true );
+ $offset=$job->id;
+ $status = $job->run();
+ $t = microtime( true ) - $t;
+ $timeMs = intval( $t * 1000 );
+ if ( !$status ) {
+ $this->runJobsLog( $job->toString() . "
t=$timeMs error={$job->error}" );
+ } else {
+ $this->runJobsLog( $job->toString() . "
t=$timeMs good" );
+ }
+ if ( $maxJobs && ++$n > $maxJobs ) {
+ break 2;
+ }
+ }
+ }
+ }
+
+ /**
+ * Log the job message
+ * @param $msg String The message to log
+ */
+ private function runJobsLog( $msg ) {
+ $this->output( wfTimestamp( TS_DB ) . " $msg\n" );
+ wfDebugLog( 'runJobs', $msg );
+ }
+}
+
+$maintClass = "RunJobs";
+require_once( DO_MAINTENANCE );
Copied: trunk/extensions/WikimediaMaintenance/jobs-loop.sh (from rev 97332,
branches/wmf/1.17wmf1/maintenance/jobs-loop.sh)
===================================================================
--- trunk/extensions/WikimediaMaintenance/jobs-loop.sh
(rev 0)
+++ trunk/extensions/WikimediaMaintenance/jobs-loop.sh 2011-09-16 22:54:57 UTC
(rev 97344)
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+trap 'kill %-; exit' SIGTERM
+[ ! -z "$1" ] && {
+ echo "starting type-specific job runner: $1"
+ type=$1
+}
+
+#types="htmlCacheUpdate sendMail enotifNotify uploadFromUrl fixDoubleRedirect
renameUser"
+types="sendMail enotifNotify uploadFromUrl fixDoubleRedirect"
+
+cd `readlink -f /usr/local/apache/common/multiversion`
+while [ 1 ];do
+ # Do the prioritised types
+ moreprio=y
+ while [ -n "$moreprio" ] ; do
+ moreprio=
+ for type in $types; do
+ db=`php -n MWScript.php nextJobDB.php --wiki=aawiki
--type="$type"`
+ if [ -n "$db" ]; then
+ echo "$db $type"
+ nice -n 20 php MWScript.php runJobs.php
--wiki="$db" --procs=5 --type="$type" --maxtime=300 &
+ wait
+ moreprio=y
+ fi
+ done
+ done
+
+ # Do the remaining types
+ db=`php -n MWScript.php nextJobDB.php --wiki=aawiki`
+
+ if [ -z "$db" ];then
+ # No jobs to do, wait for a while
+ echo "No jobs..."
+ sleep 5
+ else
+ echo "$db"
+ nice -n 20 php MWScript.php runJobs.php --wiki="$db" --procs=5
--maxtime=300 &
+ wait
+ fi
+done
Copied: trunk/extensions/WikimediaMaintenance/storage/make-all-blobs (from rev
97332, branches/wmf/1.17wmf1/maintenance/storage/make-all-blobs)
===================================================================
--- trunk/extensions/WikimediaMaintenance/storage/make-all-blobs
(rev 0)
+++ trunk/extensions/WikimediaMaintenance/storage/make-all-blobs
2011-09-16 22:54:57 UTC (rev 97344)
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [ -z $1 ];then
+ echo "Usage: make-all-blobs <server> [<table name>]"
+ exit 1
+fi
+server=$1
+if [ -z $2 ]; then
+ table=blobs
+else
+ table=$2
+fi
+
+for db in `</home/wikipedia/common/all.dblist`;do
+ echo "CREATE DATABASE IF NOT EXISTS $db" | mysql -u wikiadmin
-p`wikiadmin_pass` -h $server && \
+ sed "s/blobs\>/$table/" blobs.sql | mysql -u wikiadmin
-p`wikiadmin_pass` -h $server $db
+done
+
Copied: trunk/extensions/WikimediaMaintenance/storage/testRctComplete.php (from
rev 97332, branches/wmf/1.17wmf1/maintenance/storage/testRctComplete.php)
===================================================================
--- trunk/extensions/WikimediaMaintenance/storage/testRctComplete.php
(rev 0)
+++ trunk/extensions/WikimediaMaintenance/storage/testRctComplete.php
2011-09-16 22:54:57 UTC (rev 97344)
@@ -0,0 +1,22 @@
+<?php
+require_once( dirname(__FILE__).'/../commandLine.inc' );
+
+$bad = 0;
+$good = 0;
+foreach ( $wgLocalDatabases as $wiki ) {
+ $lb = wfGetLB( $wiki );
+ $db = $lb->getConnection( DB_SLAVE, array(), $wiki );
+ if ( $db->tableExists( 'blob_tracking' ) ) {
+ $notDone = $db->selectField( 'blob_tracking', '1',
+ array( 'bt_moved' => 0 ) );
+ if ( $notDone ) {
+ $bad++;
+ echo "$wiki\n";
+ } else {
+ $good++;
+ }
+ }
+ $lb->reuseConnection( $db );
+}
+echo "$bad wiki(s) incomplete\n";
+echo "$good wiki(s) complete\n";
_______________________________________________
MediaWiki-CVS mailing list
MediaWiki-CVS@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-cvs
