WMDE-leszek has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/358531 )
Change subject: Only load entity data TermSqlIndexBuilder when necessary ...................................................................... Only load entity data TermSqlIndexBuilder when necessary Makes TermSqlIndexBuilder only load entity data and check if any of terms are missing or are outdated when requested. This should make the "defult" rebuild faster, as it will only check the contents of wb_terms to detect possible unpopulated ID fields and/or duplicate term entries, without needing to load all the data for each processed entity. RebuildTermSqlIndex maintenance script also gets the respective option, which is turned off by default. Bug: T162533 Change-Id: Ia0409ebfd7c905b178c8ebe17945226a845e010c --- M repo/includes/Store/Sql/TermSqlIndexBuilder.php M repo/maintenance/rebuildTermSqlIndex.php 2 files changed, 44 insertions(+), 7 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Wikibase refs/changes/31/358531/1 diff --git a/repo/includes/Store/Sql/TermSqlIndexBuilder.php b/repo/includes/Store/Sql/TermSqlIndexBuilder.php index a734705..e468e00 100644 --- a/repo/includes/Store/Sql/TermSqlIndexBuilder.php +++ b/repo/includes/Store/Sql/TermSqlIndexBuilder.php @@ -13,6 +13,12 @@ /** * (Re)builds term index in the SQL table. + * This can add missing information to the SQL table like missing full entity ID. It also removes + * possible duplicate terms. + * It can also ensure that all expected entity terms are stored in the term index, i.e. add + * all possible missing terms of the given entity, and remove all possible no longer valid + * terms of the entity, even if there is no other need for rebuilding the index + * (i.e. all ID fields are populated, there are no duplicate entries). * * @license GPL-2.0+ * @author Katie Filbert < aude.w...@gmail.com > @@ -72,6 +78,11 @@ private $batchSize; /** + * @var bool + */ + private $rebuildAllEntityTerms = false; + + /** * @var int|null */ private $fromId = null; @@ -128,6 +139,15 @@ } /** + * Makes the builder rebuild all entity terms, i.e. it will check if any of entity terms + * is missing, and/or any of existing entity terms is no longer "correct". + * Missing terms will be added, and no longer expected terms will be removed. + */ + public function setRebuildAllEntityTerms() { + $this->rebuildAllEntityTerms = true; + } + + /** * @param string $entityType */ private function rebuildForEntityType( $entityType ) { @@ -163,16 +183,22 @@ private function rebuildEntityTerms( EntityId $entityId ) { $serializedId = $entityId->getSerialization(); - $entityRevision = $this->entityRevisionLookup->getEntityRevision( $entityId ); - $entity = $entityRevision->getEntity(); - - $rebuiltTerms = $this->termSqlIndex->getEntityTerms( $entity ); $existingTerms = $this->termSqlIndex->getTermsOfEntity( $entityId ); - $termsToInsert = array_udiff( $rebuiltTerms, $existingTerms, [ TermIndexEntry::class, 'compare' ] ); - $termsToDelete = array_udiff( $existingTerms, $rebuiltTerms, [ TermIndexEntry::class, 'compare' ] ); + $entityRevision = null; + $termsChanged = false; - $termsChanged = $termsToInsert || $termsToDelete; + if ( $this->rebuildAllEntityTerms ) { + $entityRevision = $this->entityRevisionLookup->getEntityRevision( $entityId ); + $entity = $entityRevision->getEntity(); + + $rebuiltTerms = $this->termSqlIndex->getEntityTerms( $entity ); + + $termsToInsert = array_udiff( $rebuiltTerms, $existingTerms, [ TermIndexEntry::class, 'compare' ] ); + $termsToDelete = array_udiff( $existingTerms, $rebuiltTerms, [ TermIndexEntry::class, 'compare' ] ); + + $termsChanged = $termsToInsert || $termsToDelete; + } $needToPopulateEntityIdColumn = !$this->readFullEntityIdColumn && $this->writeFullEntityIdColumn && @@ -194,6 +220,10 @@ return; } + if ( $entityRevision === null ) { + $entityRevision = $this->entityRevisionLookup->getEntityRevision( $entityId ); + } + $success = $this->termSqlIndex->saveTermsOfEntity( $entityRevision->getEntity() ); if ( !$success ) { diff --git a/repo/maintenance/rebuildTermSqlIndex.php b/repo/maintenance/rebuildTermSqlIndex.php index 665ee32..7e3a448 100644 --- a/repo/maintenance/rebuildTermSqlIndex.php +++ b/repo/maintenance/rebuildTermSqlIndex.php @@ -39,6 +39,9 @@ false, true ); + $this->addOption( + 'rebuild-all-terms', 'Rebuilds all terms of the entity (requires loading data of each processed entity)' + ); $this->addOption( 'from-id', "First row (page id) to start updating from", false, true ); } @@ -57,6 +60,7 @@ private function getTermIndexBuilder() { $batchSize = (int)$this->getOption( 'batch-size', 1000 ); $fromId = $this->getOption( 'from-id', null ); + $rebuildAllEntityTerms = $this->getOption( 'rebuild-all-terms', false ); $wikibaseRepo = WikibaseRepo::getDefaultInstance(); $idParser = $wikibaseRepo->getEntityIdParser(); @@ -91,6 +95,9 @@ if ( $fromId !== null ) { $builder->setFromId( (int)$fromId ); } + if ( $rebuildAllEntityTerms ) { + $builder->setRebuildAllEntityTerms(); + } return $builder; } -- To view, visit https://gerrit.wikimedia.org/r/358531 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ia0409ebfd7c905b178c8ebe17945226a845e010c Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/extensions/Wikibase Gerrit-Branch: master Gerrit-Owner: WMDE-leszek <leszek.mani...@wikimedia.de> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits