[MediaWiki-commits] [Gerrit] jobqueue: improved performance of JobQueueGroup::getQueuesWi... - change (mediawiki/core)
jenkins-bot has submitted this change and it was merged.
Change subject: jobqueue: improved performance of
JobQueueGroup::getQueuesWithJobs()
..
jobqueue: improved performance of JobQueueGroup::getQueuesWithJobs()
* Also added a JobQueueGroup::getQueueSizes() function.
This function is now used by the SiteStats to make is
meaningful for when queues are not in the DB. This number
is already exposed via the API.
bug: 45072
bug: 50635
bug: 9518
Change-Id: I75c16ffa14c963e7f8fb7cb390e6cc4cde0a5804
---
M includes/SiteStats.php
M includes/job/JobQueue.php
M includes/job/JobQueueDB.php
M includes/job/JobQueueFederated.php
M includes/job/JobQueueGroup.php
M includes/job/JobQueueRedis.php
6 files changed, 216 insertions(+), 8 deletions(-)
Approvals:
Tim Starling: Looks good to me, approved
jenkins-bot: Verified
diff --git a/includes/SiteStats.php b/includes/SiteStats.php
index 6e2359a..355993c 100644
--- a/includes/SiteStats.php
+++ b/includes/SiteStats.php
@@ -189,7 +189,7 @@
static function jobs() {
if ( !isset( self::$jobs ) ) {
$dbr = wfGetDB( DB_SLAVE );
- self::$jobs = $dbr->estimateRowCount( 'job' );
+ self::$jobs = array_sum(
JobQueueGroup::singleton()->getQueueSizes() );
/* Zero rows still do single row read for row that
doesn't exist, but people are annoyed by that */
if ( self::$jobs == 1 ) {
self::$jobs = 0;
diff --git a/includes/job/JobQueue.php b/includes/job/JobQueue.php
index 3e94b13..81e7b73 100644
--- a/includes/job/JobQueue.php
+++ b/includes/job/JobQueue.php
@@ -617,6 +617,63 @@
}
/**
+* Do not use this function outside of JobQueue/JobQueueGroup
+*
+* @return string
+* @since 1.22
+*/
+ public function getCoalesceLocationInternal() {
+ return null;
+ }
+
+ /**
+* Check whether each of the given queues are empty.
+* This is used for batching checks for queues stored at the same place.
+*
+* @param array $types List of queues types
+* @return array|null (list of non-empty queue types) or null if
unsupported
+* @throws MWException
+* @since 1.22
+*/
+ final public function getSiblingQueuesWithJobs( array $types ) {
+ $section = new ProfileSection( __METHOD__ );
+ return $this->doGetSiblingQueuesWithJobs( $types );
+ }
+
+ /**
+* @see JobQueue::getSiblingQueuesWithJobs()
+* @param array $types List of queues types
+* @return array|null (list of queue types) or null if unsupported
+*/
+ protected function doGetSiblingQueuesWithJobs( array $types ) {
+ return null; // not supported
+ }
+
+ /**
+* Check the size of each of the given queues.
+* For queues not served by the same store as this one, 0 is returned.
+* This is used for batching checks for queues stored at the same place.
+*
+* @param array $types List of queues types
+* @return array|null (job type => whether queue is empty) or null if
unsupported
+* @throws MWException
+* @since 1.22
+*/
+ final public function getSiblingQueueSizes( array $types ) {
+ $section = new ProfileSection( __METHOD__ );
+ return $this->doGetSiblingQueueSizes( $types );
+ }
+
+ /**
+* @see JobQueue::getSiblingQueuesSize()
+* @param array $types List of queues types
+* @return array|null (list of queue types) or null if unsupported
+*/
+ protected function doGetSiblingQueueSizes( array $types ) {
+ return null; // not supported
+ }
+
+ /**
* Call wfIncrStats() for the queue overall and for the queue type
*
* @param string $key Event type
diff --git a/includes/job/JobQueueDB.php b/includes/job/JobQueueDB.php
index 3fa0655..2052fc1 100644
--- a/includes/job/JobQueueDB.php
+++ b/includes/job/JobQueueDB.php
@@ -604,6 +604,34 @@
}
}
+ public function getCoalesceLocationInternal() {
+ return $this->cluster ? "DBCluster:{$this->cluster}" :
"LBFactory:{$this->wiki}";
+ }
+
+ protected function doGetSiblingQueuesWithJobs( array $types ) {
+ list( $dbr, $scope ) = $this->getSlaveDB();
+ $res = $dbr->select( 'job', 'DISTINCT job_cmd',
+ array( 'job_cmd' => $types ), __METHOD__ );
+
+ $types = array();
+ foreach ( $res as $row ) {
+ $types[] = $row->job_cmd;
+ }
+ return $types;
+ }
+
+ protected function doGetSiblingQueueSizes( array $types ) {
+
[MediaWiki-commits] [Gerrit] jobqueue: improved performance of JobQueueGroup::getQueuesWi... - change (mediawiki/core)
Aaron Schulz has uploaded a new change for review.
https://gerrit.wikimedia.org/r/71966
Change subject: jobqueue: improved performance of
JobQueueGroup::getQueuesWithJobs()
..
jobqueue: improved performance of JobQueueGroup::getQueuesWithJobs()
* Also added a JobQueueGroup::getQueueSizes() function.
This function is now used by the API to make is useful
for when queues are not in the DB.
bug: 45072
bug: 50635
bug: 9518
Change-Id: I75c16ffa14c963e7f8fb7cb390e6cc4cde0a5804
---
M includes/SiteStats.php
M includes/job/JobQueue.php
M includes/job/JobQueueDB.php
M includes/job/JobQueueFederated.php
M includes/job/JobQueueGroup.php
M includes/job/JobQueueRedis.php
6 files changed, 216 insertions(+), 8 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core
refs/changes/66/71966/1
diff --git a/includes/SiteStats.php b/includes/SiteStats.php
index 199c64f..78bb6cc 100644
--- a/includes/SiteStats.php
+++ b/includes/SiteStats.php
@@ -189,7 +189,7 @@
static function jobs() {
if ( !isset( self::$jobs ) ) {
$dbr = wfGetDB( DB_SLAVE );
- self::$jobs = $dbr->estimateRowCount( 'job' );
+ self::$jobs = array_sum(
JobQueueGroup::singleton()->getQueueSizes() );
/* Zero rows still do single row read for row that
doesn't exist, but people are annoyed by that */
if ( self::$jobs == 1 ) {
self::$jobs = 0;
diff --git a/includes/job/JobQueue.php b/includes/job/JobQueue.php
index 3295c24..2fd26fc 100644
--- a/includes/job/JobQueue.php
+++ b/includes/job/JobQueue.php
@@ -617,6 +617,63 @@
}
/**
+* Do not use this function outside of JobQueue/JobQueueGroup
+*
+* @return string
+* @since 1.22
+*/
+ public function getCoalesceLocationInternal() {
+ return null;
+ }
+
+ /**
+* Check whether each of the given queues are empty.
+* This is used for batching checks for queues stored at the same place.
+*
+* @param array $types List of queues types
+* @return array|null (list of non-empty queue types) or null if
unsupported
+* @throws MWException
+* @since 1.22
+*/
+ final public function getSiblingQueuesNonEmpty( array $types ) {
+ $section = new ProfileSection( __METHOD__ );
+ return $this->doGetSiblingQueuesNonEmpty( $types );
+ }
+
+ /**
+* @see JobQueue::getSiblingQueuesNonEmpty()
+* @param array $types List of queues types
+* @return array|null (list of queue types) or null if unsupported
+*/
+ protected function doGetSiblingQueuesNonEmpty( array $types ) {
+ return null; // not supported
+ }
+
+ /**
+* Check the size of each of the given queues.
+* For queues not served by the same store as this one, 0 is returned.
+* This is used for batching checks for queues stored at the same place.
+*
+* @param array $types List of queues types
+* @return array|null (job type => whether queue is empty) or null if
unsupported
+* @throws MWException
+* @since 1.22
+*/
+ final public function getSiblingQueueSizes( array $types ) {
+ $section = new ProfileSection( __METHOD__ );
+ return $this->doGetSiblingQueueSizes( $types );
+ }
+
+ /**
+* @see JobQueue::getSiblingQueuesSize()
+* @param array $types List of queues types
+* @return array|null (list of queue types) or null if unsupported
+*/
+ protected function doGetSiblingQueueSizes( array $types ) {
+ return null; // not supported
+ }
+
+ /**
* Call wfIncrStats() for the queue overall and for the queue type
*
* @param string $key Event type
diff --git a/includes/job/JobQueueDB.php b/includes/job/JobQueueDB.php
index 56da4f3..77011ad 100644
--- a/includes/job/JobQueueDB.php
+++ b/includes/job/JobQueueDB.php
@@ -571,6 +571,34 @@
);
}
+ public function getCoalesceLocationInternal() {
+ return $this->cluster ? "DBCluster:{$this->cluster}" :
"LBFactory:{$this->wiki}";
+ }
+
+ protected function doGetSiblingQueuesNonEmpty( array $types ) {
+ list( $dbr, $scope ) = $this->getSlaveDB();
+ $res = $dbr->select( 'job', 'DISTINCT job_cmd',
+ array( 'job_cmd' => $types ), __METHOD__ );
+
+ $types = array();
+ foreach ( $res as $row ) {
+ $types[] = $row->job_cmd;
+ }
+ return $types;
+ }
+
+ protected function doGetSiblingQueueSizes( array $types ) {
+ list( $dbr, $sc
