Aaron Schulz has uploaded a new change for review. https://gerrit.wikimedia.org/r/51789
Change subject: [JobQueue] Throttle refreshLinks2 jobs based on finishing the refreshLinks jobs. ...................................................................... [JobQueue] Throttle refreshLinks2 jobs based on finishing the refreshLinks jobs. * This should lower the rate of queue activity when major templates change. * Also fixed a broken array_diff() call in nextJobDB.php. * Additionally removed checkJob from nextJobDB.php. This is redundant to runJobs.php de-listing queues via JobQueueGroup::pop() when the queue is empty. Change-Id: I1518a0de9e7ada22350d9993dd7ffe5f2ce23745 --- M includes/DefaultSettings.php M includes/job/JobQueueGroup.php M maintenance/nextJobDB.php 3 files changed, 22 insertions(+), 15 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/89/51789/1 diff --git a/includes/DefaultSettings.php b/includes/DefaultSettings.php index 7aad30c..e9a3a16 100644 --- a/includes/DefaultSettings.php +++ b/includes/DefaultSettings.php @@ -5546,6 +5546,8 @@ /** * Map of job types to configuration arrays. + * This determines which queue class and storage system is used for each job type. + * Job types that do not have explicit configuration will use the 'default' config. * These settings should be global to all wikis. */ $wgJobTypeConf = array( diff --git a/includes/job/JobQueueGroup.php b/includes/job/JobQueueGroup.php index 23a5494..dae026c 100644 --- a/includes/job/JobQueueGroup.php +++ b/includes/job/JobQueueGroup.php @@ -237,6 +237,23 @@ } /** + * Check if jobs should not be popped of a queue right now. + * This is only used for performance, such as to avoid spamming + * the queue with many sub-jobs before they actually get run. + * + * @param $type string + * @return bool + */ + public function isQueueDeprioritized( $type ) { + if ( $type === 'refreshLinks2' ) { + // Don't keep converting refreshLinks2 => refreshLinks jobs if the + // later jobs have not been done yet. This helps throttle queue spam. + return !$this->get( 'refreshLinks' )->isEmpty(); + } + return false; + } + + /** * Execute any due periodic queue maintenance tasks for all queues. * * A task is "due" if the time ellapsed since the last run is greater than diff --git a/maintenance/nextJobDB.php b/maintenance/nextJobDB.php index f5bf4da..1be5146 100644 --- a/maintenance/nextJobDB.php +++ b/maintenance/nextJobDB.php @@ -77,10 +77,9 @@ return; // no jobs for this type } - list( $type, $db ) = $candidates[ mt_rand( 0, count( $candidates ) - 1 ) ]; - if ( !$this->checkJob( $type, $db ) ) { // queue is actually empty? - $pendingDBs[$type] = array_diff( $pendingDBs[$type], $db ); - JobQueueAggregator::singleton()->notifyQueueEmpty( $db, $type ); + list( $type, $db ) = $candidates[mt_rand( 0, count( $candidates ) - 1 )]; + if ( JobQueueGroup::singleton( $db )->isQueueDeprioritized( $type ) ) { + $pendingDBs[$type] = array_diff( $pendingDBs[$type], array( $db ) ); $again = true; } } while ( $again ); @@ -90,17 +89,6 @@ } else { $this->output( $db . "\n" ); } - } - - /** - * Check if the specified database has a job of the specified type in it. - * The type may be false to indicate "all". - * @param $type string - * @param $dbName string - * @return bool - */ - private function checkJob( $type, $dbName ) { - return !JobQueueGroup::singleton( $dbName )->get( $type )->isEmpty(); } /** -- To view, visit https://gerrit.wikimedia.org/r/51789 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I1518a0de9e7ada22350d9993dd7ffe5f2ce23745 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: master Gerrit-Owner: Aaron Schulz <asch...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits