jenkins-bot has submitted this change and it was merged.
Change subject: Make use of EntityPrefetcher in dumps, wbgetentities
......................................................................
Make use of EntityPrefetcher in dumps, wbgetentities
And implement a NullEntityPrefetcher for usage in places
we don't need to actually do any prefetching.
Change-Id: Idb2e36a1fe255334253005f6d6416e26aacb940e
---
M client/includes/store/ClientStore.php
M client/includes/store/sql/DirectSqlStore.php
M client/tests/phpunit/MockClientStore.php
M client/tests/phpunit/includes/store/sql/DirectSqlStoreTest.php
A lib/includes/store/NullEntityPrefetcher.php
M repo/includes/Dumpers/DumpGenerator.php
M repo/includes/Dumpers/JsonDumpGenerator.php
M repo/includes/Dumpers/RdfDumpGenerator.php
M repo/includes/api/GetEntities.php
M repo/includes/store/Store.php
M repo/includes/store/sql/SqlStore.php
M repo/maintenance/dumpJson.php
M repo/maintenance/dumpRdf.php
M repo/tests/phpunit/includes/Dumpers/JsonDumpGeneratorTest.php
M repo/tests/phpunit/includes/Dumpers/RdfDumpGeneratorTest.php
15 files changed, 165 insertions(+), 21 deletions(-)
Approvals:
Daniel Kinzler: Looks good to me, approved
jenkins-bot: Verified
diff --git a/client/includes/store/ClientStore.php
b/client/includes/store/ClientStore.php
index 5ab3f1c..bf63616 100644
--- a/client/includes/store/ClientStore.php
+++ b/client/includes/store/ClientStore.php
@@ -113,4 +113,14 @@
* @since 0.2
*/
public function rebuild();
+
+ /**
+ * Returns an EntityPrefetcher which can be used to prefetch a list of
entity
+ * ids in case we need to for example load a batch of entity ids.
+ *
+ * @since 0.5
+ *
+ * @return EntityPrefetcher
+ */
+ public function getEntityPrefetcher();
}
diff --git a/client/includes/store/sql/DirectSqlStore.php
b/client/includes/store/sql/DirectSqlStore.php
index ac7d5c5..dbbf3f4 100644
--- a/client/includes/store/sql/DirectSqlStore.php
+++ b/client/includes/store/sql/DirectSqlStore.php
@@ -144,6 +144,11 @@
private $subscriptionManager = null;
/**
+ * @var PrefetchingWikiPageEntityMetaDataAccessor|null
+ */
+ private $entityPrefetcher = null;
+
+ /**
* @var string
*/
private $siteId;
@@ -315,10 +320,7 @@
// NOTE: Keep cache key in sync with
SqlStore::newEntityRevisionLookup in WikibaseRepo
$cacheKeyPrefix = $this->cacheKeyPrefix .
':WikiPageEntityRevisionLookup';
- $metaDataFetcher = new
PrefetchingWikiPageEntityMetaDataAccessor(
- new WikiPageEntityMetaDataLookup(
$this->entityIdParser, $this->repoWiki )
- );
-
+ $metaDataFetcher = $this->getEntityPrefetcher();
$rawLookup = new WikiPageEntityRevisionLookup(
$this->contentCodec,
$metaDataFetcher,
@@ -451,4 +453,16 @@
return $this->propertyInfoTable;
}
+ /**
+ * @return PrefetchingWikiPageEntityMetaDataAccessor
+ */
+ public function getEntityPrefetcher() {
+ if ( $this->entityPrefetcher === null ) {
+ $this->entityPrefetcher = new
PrefetchingWikiPageEntityMetaDataAccessor(
+ new WikiPageEntityMetaDataLookup(
$this->entityIdParser )
+ );
+ }
+
+ return $this->entityPrefetcher;
+ }
}
diff --git a/client/tests/phpunit/MockClientStore.php
b/client/tests/phpunit/MockClientStore.php
index 6705537..a6aaa73 100644
--- a/client/tests/phpunit/MockClientStore.php
+++ b/client/tests/phpunit/MockClientStore.php
@@ -11,6 +11,7 @@
use Wikibase\ClientStore;
use Wikibase\Lib\Store\EntityLookup;
use Wikibase\Lib\Store\EntityRevisionLookup;
+use Wikibase\Lib\Store\NullEntityPrefetcher;
use Wikibase\Lib\Store\SiteLinkLookup;
use Wikibase\PropertyInfoStore;
use Wikibase\PropertyLabelResolver;
@@ -177,4 +178,13 @@
return self::$propertyInfoStore;
}
+ /**
+ * @see ClientStore::getEntityPrefetcher
+ *
+ * @return EntityPrefetcher
+ */
+ public function getEntityPrefetcher() {
+ return new NullEntityPrefetcher();
+ }
+
}
diff --git a/client/tests/phpunit/includes/store/sql/DirectSqlStoreTest.php
b/client/tests/phpunit/includes/store/sql/DirectSqlStoreTest.php
index 7fa41fa..fc54f4f 100644
--- a/client/tests/phpunit/includes/store/sql/DirectSqlStoreTest.php
+++ b/client/tests/phpunit/includes/store/sql/DirectSqlStoreTest.php
@@ -54,6 +54,7 @@
array( 'getUsageLookup',
'Wikibase\Client\Usage\UsageLookup' ),
array( 'getSubscriptionManager',
'Wikibase\Client\Usage\SubscriptionManager' ),
array( 'getEntityIdLookup',
'Wikibase\Store\EntityIdLookup' ),
+ array( 'getEntityPrefetcher',
'Wikibase\Lib\Store\EntityPrefetcher' ),
);
}
diff --git a/lib/includes/store/NullEntityPrefetcher.php
b/lib/includes/store/NullEntityPrefetcher.php
new file mode 100644
index 0000000..dc45d57
--- /dev/null
+++ b/lib/includes/store/NullEntityPrefetcher.php
@@ -0,0 +1,36 @@
+<?php
+
+namespace Wikibase\Lib\Store;
+
+use Wikibase\DataModel\Entity\EntityId;
+
+/**
+ * No-op EntityPrefetcher
+ *
+ * @since 0.5
+ *
+ * @license GNU GPL v2+
+ * @author Marius Hoch < h...@online.de >
+ */
+class NullEntityPrefetcher implements EntityPrefetcher {
+
+ /**
+ * Prefetches data for a list of entity ids.
+ *
+ * @param EntityId[] $entityIds
+ */
+ public function prefetch( array $entityIds ) {}
+
+ /**
+ * Purges prefetched data about a given entity.
+ *
+ * @param EntityId $entityId
+ */
+ public function purge( EntityId $entityId ) {}
+
+ /**
+ * Purges all prefetched data.
+ */
+ public function purgeAll() {}
+
+}
diff --git a/repo/includes/Dumpers/DumpGenerator.php
b/repo/includes/Dumpers/DumpGenerator.php
index 771f58b..1a27798 100644
--- a/repo/includes/Dumpers/DumpGenerator.php
+++ b/repo/includes/Dumpers/DumpGenerator.php
@@ -8,6 +8,7 @@
use Wikibase\Lib\Reporting\MessageReporter;
use Wikibase\Lib\Reporting\NullMessageReporter;
use Wikibase\Lib\Reporting\RethrowingExceptionHandler;
+use Wikibase\Lib\Store\EntityPrefetcher;
use Wikibase\Lib\Store\StorageException;
use Wikibase\Repo\Store\EntityIdPager;
@@ -53,6 +54,11 @@
protected $exceptionHandler;
/**
+ * @var EntityPrefetcher
+ */
+ protected $entityPrefetcher;
+
+ /**
* @var string
*/
protected $entityType;
@@ -66,16 +72,18 @@
/**
* @param resource $out
+ * @param EntityPrefetcher $entityPrefetcher
*
* @throws InvalidArgumentException
*/
- public function __construct( $out ) {
+ public function __construct( $out, EntityPrefetcher $entityPrefetcher )
{
if ( !is_resource( $out ) ) {
throw new InvalidArgumentException( '$out must be a
file handle!' );
}
$this->out = $out;
+ $this->entityPrefetcher = $entityPrefetcher;
$this->progressReporter = new NullMessageReporter();
$this->exceptionHandler = new RethrowingExceptionHandler();
}
@@ -276,11 +284,15 @@
* @param int &$dumpCount The number of entities already dumped (will
be updated).
*/
private function dumpEntities( array $entityIds, &$dumpCount ) {
+ $toLoad = array();
foreach ( $entityIds as $entityId ) {
- if ( !$this->idMatchesFilters( $entityId ) ) {
- continue;
+ if ( $this->idMatchesFilters( $entityId ) ) {
+ $toLoad[] = $entityId;
}
+ }
+ $this->entityPrefetcher->prefetch( $entityIds );
+ foreach ( $toLoad as $entityId ) {
try {
$data = $this->generateDumpForEntityId(
$entityId );
if ( !$data ) {
diff --git a/repo/includes/Dumpers/JsonDumpGenerator.php
b/repo/includes/Dumpers/JsonDumpGenerator.php
index 2a35aa7..ffdfd62 100644
--- a/repo/includes/Dumpers/JsonDumpGenerator.php
+++ b/repo/includes/Dumpers/JsonDumpGenerator.php
@@ -8,6 +8,7 @@
use Wikibase\DataModel\Entity\EntityId;
use Wikibase\Lib\Serializers\Serializer;
use Wikibase\Lib\Store\EntityLookup;
+use Wikibase\Lib\Store\EntityPrefetcher;
use Wikibase\Lib\Store\RedirectResolvingEntityLookup;
use Wikibase\Lib\Store\StorageException;
use Wikibase\Lib\Store\UnresolvedRedirectException;
@@ -47,11 +48,12 @@
* @param resource $out
* @param EntityLookup $lookup Must not resolve redirects
* @param Serializer $entitySerializer
+ * @param EntityPrefetcher $entityPrefetcher
*
* @throws InvalidArgumentException
*/
- public function __construct( $out, EntityLookup $lookup, Serializer
$entitySerializer ) {
- parent::__construct( $out );
+ public function __construct( $out, EntityLookup $lookup, Serializer
$entitySerializer, EntityPrefetcher $entityPrefetcher ) {
+ parent::__construct( $out, $entityPrefetcher );
if ( $lookup instanceof RedirectResolvingEntityLookup ) {
throw new InvalidArgumentException( '$lookup must not
resolve redirects!' );
}
diff --git a/repo/includes/Dumpers/RdfDumpGenerator.php
b/repo/includes/Dumpers/RdfDumpGenerator.php
index d71ea15..847f0b1 100644
--- a/repo/includes/Dumpers/RdfDumpGenerator.php
+++ b/repo/includes/Dumpers/RdfDumpGenerator.php
@@ -10,6 +10,7 @@
use Wikibase\DataModel\Entity\EntityId;
use Wikibase\DataModel\Entity\PropertyDataTypeLookup;
use Wikibase\Lib\Store\EntityLookup;
+use Wikibase\Lib\Store\EntityPrefetcher;
use Wikibase\Lib\Store\EntityRevisionLookup;
use Wikibase\Lib\Store\RedirectResolvingEntityLookup;
use Wikibase\Lib\Store\StorageException;
@@ -53,11 +54,12 @@
* @param resource $out
* @param EntityRevisionLookup $lookup Must not resolve redirects
* @param RdfSerializer $entitySerializer
+ * @param EntityPrefetcher $entityPrefetcher
*
* @throws InvalidArgumentException
*/
- public function __construct( $out, EntityRevisionLookup $lookup,
RdfSerializer $entitySerializer ) {
- parent::__construct( $out );
+ public function __construct( $out, EntityRevisionLookup $lookup,
RdfSerializer $entitySerializer, EntityPrefetcher $entityPrefetcher ) {
+ parent::__construct( $out, $entityPrefetcher );
if ( $lookup instanceof RedirectResolvingEntityLookup ) {
throw new InvalidArgumentException( '$lookup must not
resolve redirects!' );
}
@@ -136,7 +138,8 @@
SiteList $sites,
EntityLookup $entityLookup,
EntityRevisionLookup $entityRevisionLookup,
- PropertyDataTypeLookup $propertyLookup
+ PropertyDataTypeLookup $propertyLookup,
+ EntityPrefetcher $entityPrefetcher
) {
$rdfFormat = RdfSerializer::getRdfWriter( $format );
if( !$rdfFormat ) {
@@ -153,7 +156,7 @@
RdfProducer::PRODUCE_SITELINKS |
RdfProducer::PRODUCE_FULL_VALUES,
new HashBagOStuff()
);
- return new RdfDumpGenerator( $output, $entityRevisionLookup,
$entitySerializer );
+ return new RdfDumpGenerator( $output, $entityRevisionLookup,
$entitySerializer, $entityPrefetcher );
}
}
diff --git a/repo/includes/api/GetEntities.php
b/repo/includes/api/GetEntities.php
index 2e6bd5f..2380ad9 100644
--- a/repo/includes/api/GetEntities.php
+++ b/repo/includes/api/GetEntities.php
@@ -44,6 +44,11 @@
private $siteLinkTargetProvider;
/**
+ * @var EntityPrefetcher
+ */
+ private $entityPrefetcher;
+
+ /**
* @var string[]
*/
private $siteLinkGroups;
@@ -68,6 +73,7 @@
);
$this->siteLinkGroups =
$wikibaseRepo->getSettings()->getSetting( 'siteLinkGroups' );
+ $this->entityPrefetcher =
$wikibaseRepo->getStore()->getEntityPrefetcher();
}
/**
@@ -193,6 +199,8 @@
private function getEntityRevisionsFromEntityIds( $entityIds,
$resolveRedirects = false ) {
$revisionArray = array();
+ $this->entityPrefetcher->prefetch( $entityIds );
+
foreach ( $entityIds as $entityId ) {
$key = $entityId->getSerialization();
$entityRevision = $this->getEntityRevision( $entityId,
$resolveRedirects );
diff --git a/repo/includes/store/Store.php b/repo/includes/store/Store.php
index 2daa2ba..b549d54 100644
--- a/repo/includes/store/Store.php
+++ b/repo/includes/store/Store.php
@@ -4,6 +4,7 @@
use Wikibase\Lib\Store\EntityInfoBuilderFactory;
use Wikibase\Lib\Store\EntityLookup;
+use Wikibase\Lib\Store\EntityPrefetcher;
use Wikibase\Lib\Store\EntityRevisionLookup;
use Wikibase\Lib\Store\EntityStore;
use Wikibase\Lib\Store\EntityStoreWatcher;
@@ -138,4 +139,14 @@
*/
public function getSiteLinkConflictLookup();
+ /**
+ * Returns an EntityPrefetcher which can be used to prefetch a list of
entity
+ * ids in case we need to for example load a batch of entity ids.
+ *
+ * @since 0.5
+ *
+ * @return EntityPrefetcher
+ */
+ public function getEntityPrefetcher();
+
}
diff --git a/repo/includes/store/sql/SqlStore.php
b/repo/includes/store/sql/SqlStore.php
index 395cbf0..8e95216 100644
--- a/repo/includes/store/sql/SqlStore.php
+++ b/repo/includes/store/sql/SqlStore.php
@@ -101,6 +101,11 @@
private $termIndex = null;
/**
+ * @var PrefetchingWikiPageEntityMetaDataAccessor|null
+ */
+ private $entityPrefetcher = null;
+
+ /**
* @var string
*/
private $cacheKeyPrefix;
@@ -591,9 +596,7 @@
/** @var WikiPageEntityStore $dispatcher */
$dispatcher = $this->getEntityStoreWatcher();
- $metaDataFetcher = new
PrefetchingWikiPageEntityMetaDataAccessor(
- new WikiPageEntityMetaDataLookup( $this->entityIdParser
)
- );
+ $metaDataFetcher = $this->getEntityPrefetcher();
$dispatcher->registerWatcher( $metaDataFetcher );
$rawLookup = new WikiPageEntityRevisionLookup(
@@ -711,4 +714,17 @@
return new SiteLinkTable( 'wb_items_per_site', false );
}
+ /**
+ * @return PrefetchingWikiPageEntityMetaDataAccessor
+ */
+ public function getEntityPrefetcher() {
+ if ( $this->entityPrefetcher === null ) {
+ $this->entityPrefetcher = new
PrefetchingWikiPageEntityMetaDataAccessor(
+ new WikiPageEntityMetaDataLookup(
$this->entityIdParser )
+ );
+ }
+
+ return $this->entityPrefetcher;
+ }
+
}
diff --git a/repo/maintenance/dumpJson.php b/repo/maintenance/dumpJson.php
index 96e24b5..10b123b 100644
--- a/repo/maintenance/dumpJson.php
+++ b/repo/maintenance/dumpJson.php
@@ -30,8 +30,15 @@
);
$entitySerializer = new DispatchingEntitySerializer(
$serializerFactory, $serializerOptions );
+ $entityPrefetcher =
$this->wikibaseRepo->getStore()->getEntityPrefetcher();
- $dumper = new JsonDumpGenerator( $output, $this->entityLookup,
$entitySerializer );
+ $dumper = new JsonDumpGenerator(
+ $output,
+ $this->entityLookup,
+ $entitySerializer,
+ $entityPrefetcher
+ );
+
$dumper->setUseSnippets( (bool)$this->getOption( 'snippet',
false ) );
return $dumper;
}
diff --git a/repo/maintenance/dumpRdf.php b/repo/maintenance/dumpRdf.php
index 61f56c7..82e4ca7 100644
--- a/repo/maintenance/dumpRdf.php
+++ b/repo/maintenance/dumpRdf.php
@@ -32,7 +32,8 @@
$entityDataTitle->getCanonicalURL() . '/',
$this->wikibaseRepo->getSiteStore()->getSites(),
$this->entityLookup, $this->revisionLookup,
- $this->wikibaseRepo->getPropertyDataTypeLookup() );
+ $this->wikibaseRepo->getPropertyDataTypeLookup(),
+ $this->wikibaseRepo->getStore()->getEntityPrefetcher()
);
}
}
diff --git a/repo/tests/phpunit/includes/Dumpers/JsonDumpGeneratorTest.php
b/repo/tests/phpunit/includes/Dumpers/JsonDumpGeneratorTest.php
index 2d352c3..315e436 100644
--- a/repo/tests/phpunit/includes/Dumpers/JsonDumpGeneratorTest.php
+++ b/repo/tests/phpunit/includes/Dumpers/JsonDumpGeneratorTest.php
@@ -15,6 +15,7 @@
use Wikibase\Lib\Serializers\DispatchingEntitySerializer;
use Wikibase\Lib\Serializers\SerializationOptions;
use Wikibase\Lib\Serializers\SerializerFactory;
+use Wikibase\Lib\Store\NullEntityPrefetcher;
use Wikibase\Lib\Store\UnresolvedRedirectException;
use Wikibase\Repo\Store\EntityIdPager;
@@ -115,7 +116,12 @@
return $entities[$key];
} ) );
- return new JsonDumpGenerator( $out, $entityLookup, $serializer
);
+ return new JsonDumpGenerator(
+ $out,
+ $entityLookup,
+ $serializer,
+ new NullEntityPrefetcher()
+ );
}
/**
@@ -193,7 +199,12 @@
$out = fopen( 'php://output', 'w' );
$serializer = new DispatchingEntitySerializer(
$this->serializerFactory );
- $jsonDumper = new JsonDumpGenerator( $out, $entityLookup,
$serializer );
+ $jsonDumper = new JsonDumpGenerator(
+ $out,
+ $entityLookup,
+ $serializer,
+ new NullEntityPrefetcher()
+ );
$exceptionHandler = $this->getMock(
'Wikibase\Lib\Reporting\ExceptionHandler' );
$exceptionHandler->expects( $this->exactly( count( $ids ) ) )
diff --git a/repo/tests/phpunit/includes/Dumpers/RdfDumpGeneratorTest.php
b/repo/tests/phpunit/includes/Dumpers/RdfDumpGeneratorTest.php
index a6d575b..dcbb772 100644
--- a/repo/tests/phpunit/includes/Dumpers/RdfDumpGeneratorTest.php
+++ b/repo/tests/phpunit/includes/Dumpers/RdfDumpGeneratorTest.php
@@ -9,6 +9,7 @@
use Wikibase\DataModel\Entity\EntityId;
use Wikibase\DataModel\Entity\ItemId;
use Wikibase\DataModel\Entity\PropertyId;
+use Wikibase\Lib\Store\NullEntityPrefetcher;
use Wikibase\Dumpers\RdfDumpGenerator;
use Wikibase\EntityRevision;
use Wikibase\Test\RdfBuilderTest;
@@ -91,7 +92,8 @@
$this->getSiteList(),
$entityLookup,
$entityRevisionLookup,
- $propertyLookup
+ $propertyLookup,
+ new NullEntityPrefetcher()
);
}
--
To view, visit https://gerrit.wikimedia.org/r/203306
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Idb2e36a1fe255334253005f6d6416e26aacb940e
Gerrit-PatchSet: 4
Gerrit-Project: mediawiki/extensions/Wikibase
Gerrit-Branch: master
Gerrit-Owner: Hoo man <h...@online.de>
Gerrit-Reviewer: Addshore <addshorew...@gmail.com>
Gerrit-Reviewer: Daniel Kinzler <daniel.kinz...@wikimedia.de>
Gerrit-Reviewer: Hoo man <h...@online.de>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
