Daniel Kinzler has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/377046 )
Change subject: Allow batch sizes for different jobs to be defined separately. ...................................................................... Allow batch sizes for different jobs to be defined separately. In particular, the batch size for UpdateHtmlCacheJob can be quite large, while the batch size for the slow running RefreshLinksJob should be rather small. Bug: T173710 Change-Id: Ic095e2eba985ea3a6e51645d0be58589b326f218 --- M client/config/WikibaseClient.default.php M client/includes/Changes/InjectRCRecordsJob.php M client/includes/Changes/WikiPageUpdater.php M client/includes/WikibaseClient.php M client/tests/phpunit/includes/Changes/InjectRCRecordsJobTest.php M docs/options.wiki 6 files changed, 88 insertions(+), 30 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Wikibase refs/changes/46/377046/1 diff --git a/client/config/WikibaseClient.default.php b/client/config/WikibaseClient.default.php index f75c05c..ca7f007 100644 --- a/client/config/WikibaseClient.default.php +++ b/client/config/WikibaseClient.default.php @@ -96,8 +96,23 @@ $settings->getSetting( 'hasFullEntityIdColumn' ) : true; }, - // Database batch size in WikiPageUpdater which ChangeHandler uses - 'wikiPageUpdaterDbBatchSize' => 50, + // Batch size for UpdateHtmlCacheJob + 'purgeCacheBatchSize' => function ( SettingsArray $settings ) { + return $settings->hasSetting( 'wikiPageUpdaterDbBatchSize' ) ? + $settings->getSetting( 'wikiPageUpdaterDbBatchSize' ) : 100; + }, + + // Batch size for InjectRCRecordsJob + 'recentChangesBatchSize' => function ( SettingsArray $settings ) { + return $settings->hasSetting( 'wikiPageUpdaterDbBatchSize' ) ? + $settings->getSetting( 'wikiPageUpdaterDbBatchSize' ) : 100; + }, + + // Batch size for RefreshLinksJobs + 'refreshLinksBatchSize' => function ( SettingsArray $settings ) { + return $settings->hasSetting( 'wikiPageUpdaterDbBatchSize' ) ? + $settings->getSetting( 'wikiPageUpdaterDbBatchSize' ) : 5; + }, ]; // Some defaults depend on information not available at this time. diff --git a/client/includes/Changes/InjectRCRecordsJob.php b/client/includes/Changes/InjectRCRecordsJob.php index dd220ed..ded5067 100644 --- a/client/includes/Changes/InjectRCRecordsJob.php +++ b/client/includes/Changes/InjectRCRecordsJob.php @@ -68,21 +68,18 @@ private $stats = null; /** - * @var int Batch size for database operations - */ - private $dbBatchSize = 100; - - /** * @param Title[] $titles * @param EntityChange $change * @param array $rootJobParams + * @param int $batchSize * * @return JobSpecification */ public static function makeJobSpecification( array $titles, EntityChange $change, - array $rootJobParams = [] + array $rootJobParams = [], + $batchSize = 100 ) { $pages = []; @@ -99,7 +96,8 @@ // See JobQueueChangeNotificationSender::getJobSpecification for relevant root job parameters. $params = array_merge( $rootJobParams, [ 'change' => $changeData, - 'pages' => $pages + 'pages' => $pages, + 'batchSize' => $batchSize, ] ); return new JobSpecification( @@ -190,11 +188,11 @@ } /** - * @param int $dbBatchSize + * @return int */ - public function setDbBatchSize( $dbBatchSize ) { - Assert::parameterType( 'integer', $dbBatchSize, '$dbBatchSize' ); - $this->dbBatchSize = $dbBatchSize; + private function getBatchSize() { + $params = $this->getParams(); + return isset( $params['batchSize'] ) ? intval( $params['batchSize'] ) : 100; } /** @@ -286,7 +284,7 @@ $rc->save(); } - if ( ++$c >= $this->dbBatchSize ) { + if ( ++$c >= $this->getBatchSize() ) { $this->lbFactory->commitAndWaitForReplication( __METHOD__, $trxToken ); $trxToken = $this->lbFactory->getEmptyTransactionTicket( __METHOD__ ); $c = 0; diff --git a/client/includes/Changes/WikiPageUpdater.php b/client/includes/Changes/WikiPageUpdater.php index 6e66fcb..363cd73 100644 --- a/client/includes/Changes/WikiPageUpdater.php +++ b/client/includes/Changes/WikiPageUpdater.php @@ -10,7 +10,6 @@ use Wikibase\Client\RecentChanges\RecentChangeFactory; use Wikibase\Client\RecentChanges\RecentChangesDuplicateDetector; use Wikibase\EntityChange; -use Wikimedia\Assert\Assert; use Wikimedia\Rdbms\LBFactory; /** @@ -40,9 +39,19 @@ private $LBFactory; /** - * @var int Batch size for database operations + * @var int Batch size for UpdateHtmlCacheJob */ - private $dbBatchSize = 50; + private $purgeCacheBatchSize = 100; + + /** + * @var int Batch size for InjectRCRecordsJob + */ + private $rcBatchSize = 100; + + /** + * @var int Batch size for RefreshLinksJobs + */ + private $refreshLinksBatchSize = 5; /** * @var RecentChangesDuplicateDetector|null @@ -78,17 +87,43 @@ /** * @return int */ - public function getDbBatchSize() { - return $this->dbBatchSize; + public function getPurgeCacheBatchSize() { + return $this->purgeCacheBatchSize; } /** - * @param int $dbBatchSize + * @param int $purgeCacheBatchSize */ - public function setDbBatchSize( $dbBatchSize ) { - Assert::parameterType( 'integer', $dbBatchSize, 'dbBatchSize' ); + public function setPurgeCacheBatchSize( $purgeCacheBatchSize ) { + $this->purgeCacheBatchSize = $purgeCacheBatchSize; + } - $this->dbBatchSize = $dbBatchSize; + /** + * @return int + */ + public function getRefreshLinksBatchSize() { + return $this->refreshLinksBatchSize; + } + + /** + * @param int $refreshLinksBatchSize + */ + public function setRefreshLinksBatchSize( $refreshLinksBatchSize ) { + $this->refreshLinksBatchSize = $refreshLinksBatchSize; + } + + /** + * @return int + */ + public function getRecentChangesBatchSize() { + return $this->rcBatchSize; + } + + /** + * @param int $rcBatchSize + */ + public function setRecentChangesBatchSize( $rcBatchSize ) { + $this->rcBatchSize = $rcBatchSize; } private function incrementStats( $updateType, $delta ) { @@ -133,7 +168,7 @@ } $jobs = []; - $titleBatches = array_chunk( $titles, $this->dbBatchSize ); + $titleBatches = array_chunk( $titles, $this->getPurgeCacheBatchSize() ); /* @var Title[] $batch */ foreach ( $titleBatches as $batch ) { @@ -167,7 +202,7 @@ } $jobs = []; - $titleBatches = array_chunk( $titles, $this->dbBatchSize ); + $titleBatches = array_chunk( $titles, $this->getRefreshLinksBatchSize() ); /* @var Title[] $batch */ foreach ( $titleBatches as $batch ) { @@ -220,7 +255,12 @@ return; } - $jobSpec = InjectRCRecordsJob::makeJobSpecification( $titles, $change, $rootJobParams ); + $jobSpec = InjectRCRecordsJob::makeJobSpecification( + $titles, + $change, + $rootJobParams, + $this->getRecentChangesBatchSize() + ); $this->jobQueueGroup->lazyPush( $jobSpec ); diff --git a/client/includes/WikibaseClient.php b/client/includes/WikibaseClient.php index 4b14a38..74e5cb1 100644 --- a/client/includes/WikibaseClient.php +++ b/client/includes/WikibaseClient.php @@ -1156,7 +1156,9 @@ MediaWikiServices::getInstance()->getStatsdDataFactory() ); - $pageUpdater->setDbBatchSize( $this->settings->getSetting( 'wikiPageUpdaterDbBatchSize' ) ); + $pageUpdater->setPurgeCacheBatchSize( $this->settings->getSetting( 'purgeCacheBatchSize' ) ); + $pageUpdater->setRefreshLinksBatchSize( $this->settings->getSetting( 'refreshLinksBatchSize' ) ); + $pageUpdater->setRecentChangesBatchSize( $this->settings->getSetting( 'recentChangesBatchSize' ) ); $changeListTransformer = new ChangeRunCoalescer( $this->getStore()->getEntityRevisionLookup(), diff --git a/client/tests/phpunit/includes/Changes/InjectRCRecordsJobTest.php b/client/tests/phpunit/includes/Changes/InjectRCRecordsJobTest.php index 1636500..cd117d8 100644 --- a/client/tests/phpunit/includes/Changes/InjectRCRecordsJobTest.php +++ b/client/tests/phpunit/includes/Changes/InjectRCRecordsJobTest.php @@ -493,7 +493,8 @@ 21 => [ 0, 'Foo' ], 22 => [ 0, 'Bar' ], 23 => [ 0, 'Cuzz' ], - ] + ], + 'batchSize' => 2 ]; $job = new InjectRCRecordsJob( @@ -505,7 +506,6 @@ ); $job->setTitleFactory( $this->getTitleFactoryMock() ); - $job->setDbBatchSize( 2 ); $job->run(); } diff --git a/docs/options.wiki b/docs/options.wiki index 6cf9eb2..ddc3dec 100644 --- a/docs/options.wiki +++ b/docs/options.wiki @@ -110,4 +110,7 @@ ;sendEchoNotification: If true, allows users on the client wiki to get a notification when a page they created is connected to a repo item. This requires the Echo extension. ;echoIcon: If <code>sendEchoNotification</code> is set to <code>true</code>, you can also provide what icon the user will see. The correct syntax is <code>[ 'url' => '...' ]</code> or <code>[ 'path' => '...' ]</code> where <code>path</code> is relative to <code>$wgExtensionAssetsPath</code>. Defaults to <code>false</code> which means that there will be the default Echo icon. ;disabledUsageAspects: Array of usage aspects that should not be saved in the <code>wbc_entity_usage</code> table. This currently only supports aspect codes (like "T", "L" or "X"), but not full aspect keys (like "L.de"). -;wikiPageUpdaterDbBatchSize: Batch size in updating page table when applying changes in the client, default is 50. +;wikiPageUpdaterDbBatchSize: DEPRECATED. If set, acts as a default for purgeCacheBatchSize, recentChangesBatchSize, and refreshLinksBatchSize. +;purgeCacheBatchSize: Number of pages to process in each UpdateHtmlCacheJob, defaults to 100. +;refreshLinksBatchSize: Number of pages to process in each RefreshLinksJobs, defaults to 5. +;recentChangesBatchSize: Batch size for database operations in InjectRCRecordsJob, defaults to 100. -- To view, visit https://gerrit.wikimedia.org/r/377046 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ic095e2eba985ea3a6e51645d0be58589b326f218 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/extensions/Wikibase Gerrit-Branch: master Gerrit-Owner: Daniel Kinzler <daniel.kinz...@wikimedia.de> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits