Krinkle has uploaded a new change for review.
https://gerrit.wikimedia.org/r/252880
Change subject: [WIP] MessageBlobStore: Convert from msg_resource table to
object cache
......................................................................
[WIP] MessageBlobStore: Convert from msg_resource table to object cache
* Remove all interaction with the msg_resource table.
TODO:
* Make the retrieval more performant by adding MessageBlobStore
keys in WAN cache. Or by making MessageCache support getMulti.
Bug: T113092
Change-Id: I78bb09bcea384dd166e5c13ff9b2e882c70e4df9
---
M includes/cache/MessageBlobStore.php
M includes/resourceloader/ResourceLoaderFileModule.php
M includes/resourceloader/ResourceLoaderModule.php
M maintenance/cleanupRemovedModules.php
M tests/phpunit/includes/OutputPageTest.php
5 files changed, 124 insertions(+), 367 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core
refs/changes/80/252880/1
diff --git a/includes/cache/MessageBlobStore.php
b/includes/cache/MessageBlobStore.php
index 9d0d599..7d93756 100644
--- a/includes/cache/MessageBlobStore.php
+++ b/includes/cache/MessageBlobStore.php
@@ -31,284 +31,52 @@
* or the module definition, is changed.
*/
class MessageBlobStore {
- /**
- * In-process cache for message blobs.
- *
- * Keyed by language code, then module name.
- *
- * @var array
- */
- protected $blobCache = array();
- /* @var ResourceLoader */
- protected $resourceloader;
+ public function __construct( ResourceLoader $unused = null ) {
+ }
/**
- * @param ResourceLoader $resourceloader
+ * Get the message blob for a module
+ *
+ * @since 1.27
+ * @param ResourceLoaderModule|array $modules Array of module objects
keyed by module name
+ * @param string $lang Language code
+ * @return string|null JSON blob or null of module has no messages
*/
- public function __construct( ResourceLoader $resourceloader = null ) {
- $this->resourceloader = $resourceloader;
+ public function getBlob( ResourceLoaderModule $module, $lang ) {
+ return $this->generateMessageBlob( $module, $lang );
}
/**
* Get the message blobs for a set of modules
*
* @param ResourceLoader $resourceLoader
- * @param array $modules Array of module objects keyed by module name
+ * @param ResourceLoaderModule|array $modules Array of module objects
keyed by module name
* @param string $lang Language code
* @return array An array mapping module names to message blobs
*/
public function get( ResourceLoader $resourceLoader, $modules, $lang ) {
+ $modules = is_array( $modules ) ? $modules : array( $modules );
+
if ( !count( $modules ) ) {
return array();
}
$blobs = array();
- // Try in-process cache
- $missingFromCache = array();
- foreach ( $modules as $name => $module ) {
- if ( isset( $this->blobCache[$lang][$name] ) ) {
- $blobs[$name] = $this->blobCache[$lang][$name];
- } else {
- $missingFromCache[$name] = $module;
- }
- }
-
- // Try DB cache
- if ( $missingFromCache ) {
- $blobs += $this->getFromDB( $missingFromCache, $lang );
- }
-
- // Generate new blobs for any remaining modules and store in DB
- $missingFromDb = array_diff( array_keys( $modules ),
array_keys( $blobs ) );
- foreach ( $missingFromDb as $name ) {
- $blob = $this->insertMessageBlob( $name,
$modules[$name], $lang );
+ // Try to generate blobs for each module
+ foreach ( $module as $name => $module ) {
+ $blob = $this->generateMessageBlob( $module, $lang );
if ( $blob ) {
$blobs[$name] = $blob;
}
- }
-
- // Update in-process cache
- if ( isset( $this->blobCache[$lang] ) ) {
- $this->blobCache[$lang] += $blobs;
- } else {
- $this->blobCache[$lang] = $blobs;
}
return $blobs;
}
/**
- * Generate and insert a new message blob. If the blob was already
- * present, it is not regenerated; instead, the preexisting blob
- * is fetched and returned.
- *
- * @param string $name Module name
- * @param ResourceLoaderModule $module
- * @param string $lang Language code
- * @return mixed Message blob or false if the module has no messages
- */
- public function insertMessageBlob( $name, ResourceLoaderModule $module,
$lang ) {
- $blob = $this->generateMessageBlob( $module, $lang );
-
- if ( !$blob ) {
- return false;
- }
-
- try {
- $dbw = wfGetDB( DB_MASTER );
- $success = $dbw->insert( 'msg_resource', array(
- 'mr_lang' => $lang,
- 'mr_resource' => $name,
- 'mr_blob' => $blob,
- 'mr_timestamp' => $dbw->timestamp()
- ),
- __METHOD__,
- array( 'IGNORE' )
- );
-
- if ( $success && $dbw->affectedRows() == 0 ) {
- // Blob was already present, fetch it
- $blob = $dbw->selectField( 'msg_resource',
'mr_blob', array(
- 'mr_resource' => $name,
- 'mr_lang' => $lang,
- ),
- __METHOD__
- );
- }
- } catch ( DBError $e ) {
- wfDebug( __METHOD__ . " failed to update DB: $e\n" );
- }
- return $blob;
- }
-
- /**
- * Update the message blob for a given module in a given language
- *
- * @param string $name Module name
- * @param ResourceLoaderModule $module
- * @param string $lang Language code
- * @return string|null Regenerated message blob, or null if there was
no blob for
- * the given module/language pair.
- */
- public function updateModule( $name, ResourceLoaderModule $module,
$lang ) {
- $dbw = wfGetDB( DB_MASTER );
- $row = $dbw->selectRow( 'msg_resource', 'mr_blob',
- array( 'mr_resource' => $name, 'mr_lang' => $lang ),
- __METHOD__
- );
- if ( !$row ) {
- return null;
- }
-
- $newBlob = $this->generateMessageBlob( $module, $lang );
-
- try {
- $newRow = array(
- 'mr_resource' => $name,
- 'mr_lang' => $lang,
- 'mr_blob' => $newBlob,
- 'mr_timestamp' => $dbw->timestamp()
- );
-
- $dbw->replace( 'msg_resource',
- array( array( 'mr_resource', 'mr_lang' ) ),
- $newRow, __METHOD__
- );
- } catch ( Exception $e ) {
- wfDebug( __METHOD__ . " failed to update DB: $e\n" );
- }
- return $newBlob;
- }
-
- /**
- * Update a single message in all message blobs it occurs in.
- *
- * @param string $key Message key
- */
- public function updateMessage( $key ) {
- try {
- $dbw = wfGetDB( DB_MASTER );
-
- // Keep running until the updates queue is empty.
- // Due to update conflicts, the queue might not be
emptied
- // in one iteration.
- $updates = null;
- do {
- $updates = $this->getUpdatesForMessage( $key,
$updates );
-
- foreach ( $updates as $k => $update ) {
- // Update the row on the condition that
it
- // didn't change since we fetched it by
putting
- // the timestamp in the WHERE clause.
- $success = $dbw->update( 'msg_resource',
- array(
- 'mr_blob' =>
$update['newBlob'],
- 'mr_timestamp' =>
$dbw->timestamp() ),
- array(
- 'mr_resource' =>
$update['resource'],
- 'mr_lang' =>
$update['lang'],
- 'mr_timestamp' =>
$update['timestamp'] ),
- __METHOD__
- );
-
- // Only requeue conflicted updates.
- // If update() returned false, don't
retry, for
- // fear of getting into an infinite loop
- if ( !( $success &&
$dbw->affectedRows() == 0 ) ) {
- // Not conflicted
- unset( $updates[$k] );
- }
- }
- } while ( count( $updates ) );
-
- } catch ( Exception $e ) {
- wfDebug( __METHOD__ . " failed to update DB: $e\n" );
- }
- }
-
- public function clear() {
- try {
- // Not using TRUNCATE, because that needs extra
permissions,
- // which maybe not granted to the database user.
- $dbw = wfGetDB( DB_MASTER );
- $dbw->delete( 'msg_resource', '*', __METHOD__ );
- } catch ( Exception $e ) {
- wfDebug( __METHOD__ . " failed to update DB: $e\n" );
- }
- }
-
- /**
- * @return ResourceLoader
- */
- protected function getResourceLoader() {
- // For back-compat this class supports instantiation without
passing ResourceLoader
- // Lazy-initialise this property because most callers don't
need it.
- if ( $this->resourceloader === null ) {
- wfDebug( __CLASS__ . ' created without a ResourceLoader
instance' );
- $this->resourceloader = new ResourceLoader();
- }
-
- return $this->resourceloader;
- }
-
- /**
- * Create an update queue for updateMessage()
- *
- * @param string $key Message key
- * @param array $prevUpdates Updates queue to refresh or null to build
a fresh update queue
- * @return array Updates queue
- */
- private function getUpdatesForMessage( $key, $prevUpdates = null ) {
- $dbw = wfGetDB( DB_MASTER );
-
- if ( is_null( $prevUpdates ) ) {
- $rl = $this->getResourceLoader();
- $moduleNames = $rl->getModulesByMessage( $key );
- // Fetch all blobs referencing $key
- $res = $dbw->select(
- array( 'msg_resource' ),
- array( 'mr_resource', 'mr_lang', 'mr_blob',
'mr_timestamp' ),
- array(
- 'mr_resource' => $moduleNames,
- ),
- __METHOD__
- );
- } else {
- // Refetch the blobs referenced by $prevUpdates
-
- // Reorganize the (resource, lang) pairs in the format
- // expected by makeWhereFrom2d()
- $twoD = array();
-
- foreach ( $prevUpdates as $update ) {
- $twoD[$update['resource']][$update['lang']] =
true;
- }
-
- $res = $dbw->select( 'msg_resource',
- array( 'mr_resource', 'mr_lang', 'mr_blob',
'mr_timestamp' ),
- $dbw->makeWhereFrom2d( $twoD, 'mr_resource',
'mr_lang' ),
- __METHOD__
- );
- }
-
- // Build the new updates queue
- $updates = array();
-
- foreach ( $res as $row ) {
- $updates[] = array(
- 'resource' => $row->mr_resource,
- 'lang' => $row->mr_lang,
- 'timestamp' => $row->mr_timestamp,
- 'newBlob' => $this->reencodeBlob(
$row->mr_blob, $key, $row->mr_lang )
- );
- }
-
- return $updates;
- }
-
- /**
+ * @since 1.27
* @param string $key Message key
* @param string $lang Language code
* @return string
@@ -322,75 +90,77 @@
}
/**
- * Reencode a message blob with the updated value for a message
- *
- * @param string $blob Message blob (JSON object)
- * @param string $key Message key
- * @param string $lang Language code
- * @return string Message blob with $key replaced with its new value
- */
- private function reencodeBlob( $blob, $key, $lang ) {
- $decoded = FormatJson::decode( $blob, true );
- $decoded[$key] = $this->fetchMessage( $key, $lang );
- return FormatJson::encode( (object)$decoded );
- }
-
- /**
- * Get the message blobs for a set of modules from the database.
- * Modules whose blobs are not in the database are silently dropped.
- *
- * @param array $modules Array of module objects by name
- * @param string $lang Language code
- * @throws MWException
- * @return array Array mapping module names to blobs
- */
- private function getFromDB( $modules, $lang ) {
- if ( !count( $modules ) ) {
- return array();
- }
-
- $retval = array();
- $dbr = wfGetDB( DB_SLAVE );
- $res = $dbr->select( 'msg_resource',
- array( 'mr_blob', 'mr_resource', 'mr_timestamp' ),
- array( 'mr_resource' => array_keys( $modules ),
'mr_lang' => $lang ),
- __METHOD__
- );
-
- foreach ( $res as $row ) {
- if ( !isset( $modules[ $row->mr_resource ] ) {
- // This shouldn't be possible
- throw new MWException( __METHOD__ . ' passed an
invalid module name' );
- }
- $module = $modules[ $row->mr_resource ];
-
- // Update the module's blob if the list of messages
changed
- $blobKeys = array_keys( FormatJson::decode(
$row->mr_blob, true ) );
- $moduleMsgs = array_values( array_unique(
$module->getMessages() ) );
- if ( $blobKeys !== $moduleMsgs ) {
- $retval[$row->mr_resource] =
$this->updateModule( $row->mr_resource, $module, $lang );
- } else {
- $retval[$row->mr_resource] = $row->mr_blob;
- }
- }
-
- return $retval;
- }
-
- /**
* Generate the message blob for a given module in a given language.
*
* @param ResourceLoaderModule $module
* @param string $lang Language code
- * @return string JSON object
+ * @return string|null JSON blob or null if the module has no messages
*/
private function generateMessageBlob( ResourceLoaderModule $module,
$lang ) {
+ if ( !$module->getMessages() ) {
+ return null;
+ }
$messages = array();
-
foreach ( $module->getMessages() as $key ) {
$messages[$key] = $this->fetchMessage( $key, $lang );
}
return FormatJson::encode( (object)$messages );
}
+
+ /**
+ * Update a single message in all message blobs it occurs in.
+ *
+ * Called from MessageCache when a message value is edited.
+ *
+ * @param string $key Message key
+ */
+ public function updateMessage( $key ) {
+ // TODO:
+ // - Find which modules use the message
+ // - Purge those keys
+ }
+
+
+ /**
+ * Generate the message blob for a module/language pair.
+ *
+ * Previously used to populate a cache table in the database.
+ *
+ * @deprecated since 1.27 Obsolete
+ * @param string $name Module name
+ * @param ResourceLoaderModule $module
+ * @param string $lang Language code
+ * @return string|bool Message blob or false if the module has no
messages
+ */
+ public function insertMessageBlob( $name, ResourceLoaderModule $module,
$lang ) {
+ return $this->generateMessageBlob( $module, $lang ) ?: false;
+ }
+
+ /**
+ * Update the message blob in the database for a module/language pair.
+ *
+ * @deprecated since 1.27 Obsolete
+ * @param string $name Module name
+ * @param ResourceLoaderModule $module
+ * @param string $lang Language code
+ * @return null
+ */
+ public function updateModule( $name, ResourceLoaderModule $module,
$lang ) {
+ return null;
+ }
+
+ /**
+ * @deprecated since 1.27 Obsolete
+ */
+ public function clear() {
+ try {
+ // Not using TRUNCATE, because that needs extra
permissions,
+ // which maybe not granted to the database user.
+ $dbw = wfGetDB( DB_MASTER );
+ $dbw->delete( 'msg_resource', '*', __METHOD__ );
+ } catch ( Exception $e ) {
+ wfDebug( __METHOD__ . " failed to update DB: $e\n" );
+ }
+ }
}
diff --git a/includes/resourceloader/ResourceLoaderFileModule.php
b/includes/resourceloader/ResourceLoaderFileModule.php
index 1421b10..8e392c1 100644
--- a/includes/resourceloader/ResourceLoaderFileModule.php
+++ b/includes/resourceloader/ResourceLoaderFileModule.php
@@ -583,7 +583,7 @@
$summary[] = array(
'options' => $options,
'fileHashes' => $this->getFileHashes( $context ),
- 'msgBlobMtime' => $this->getMsgBlobMtime(
$context->getLanguage() ),
+ 'messageBlob' => $this->getMessageBlob(
$context->getLanguage() ),
);
return $summary;
}
diff --git a/includes/resourceloader/ResourceLoaderModule.php
b/includes/resourceloader/ResourceLoaderModule.php
index fcda87b..45ca9df 100644
--- a/includes/resourceloader/ResourceLoaderModule.php
+++ b/includes/resourceloader/ResourceLoaderModule.php
@@ -60,8 +60,8 @@
// In-object cache for file dependencies
protected $fileDeps = array();
- // In-object cache for message blob mtime
- protected $msgBlobMtime = array();
+ // In-object cache for message blob (keyed by language)
+ protected $msgBlobs = array();
// In-object cache for version hash
protected $versionHash = array();
// In-object cache for module content
@@ -453,45 +453,18 @@
}
/**
- * Get the last modification timestamp of the messages in this module
for a given language.
+ * Get the hash of the message blob.
+ *
+ * @since 1.27
* @param string $lang Language code
- * @return int UNIX timestamp
+ * @return string JSON blob
*/
- public function getMsgBlobMtime( $lang ) {
- if ( !isset( $this->msgBlobMtime[$lang] ) ) {
- if ( !count( $this->getMessages() ) ) {
- return 1;
- }
-
- $dbr = wfGetDB( DB_SLAVE );
- $msgBlobMtime = $dbr->selectField( 'msg_resource',
- 'mr_timestamp',
- array(
- 'mr_resource' => $this->getName(),
- 'mr_lang' => $lang
- ),
- __METHOD__
- );
- // If no blob was found, but the module does have
messages, that means we need
- // to regenerate it. Return NOW
- if ( $msgBlobMtime === false ) {
- $msgBlobMtime = wfTimestampNow();
- }
- $this->msgBlobMtime[$lang] = wfTimestamp( TS_UNIX,
$msgBlobMtime );
+ protected function getMessageBlob( $lang ) {
+ if ( !isset( $this->msgBlobs[$lang] ) ) {
+ $store =
$context->getResourceLoader()->getMessageBlobStore();
+ $this->msgBlobs[$lang] = $store->getBlob( $this, $lang
);
}
- return $this->msgBlobMtime[$lang];
- }
-
- /**
- * Set in-object cache for message blob time.
- *
- * This is used to retrieve data in batches. See
ResourceLoader::preloadModuleInfo().
- *
- * @param string $lang Language code
- * @param int $mtime UNIX timestamp
- */
- public function setMsgBlobMtime( $lang, $mtime ) {
- $this->msgBlobMtime[$lang] = $mtime;
+ return $this->msgBlobs[$lang];
}
/**
@@ -607,13 +580,9 @@
}
// Messages
- $blobs = $rl->getMessageBlobStore()->get(
- $rl,
- array( $this->getName() => $this ),
- $context->getLanguage()
- );
- if ( isset( $blobs[$this->getName()] ) ) {
- $content['messagesBlob'] = $blobs[$this->getName()];
+ $blob = $this->getMessageBlob( $context->getLanguage() );
+ if ( $blob ) {
+ $content['messagesBlob'] = $blob;
}
$templates = $this->getTemplates();
@@ -746,7 +715,7 @@
* A number of utility methods are available to help you gather data.
These are not
* called by default and must be included by the subclass'
getDefinitionSummary().
*
- * - getMsgBlobMtime()
+ * - getMessageBlob()
*
* @since 1.23
* @param ResourceLoaderContext $context
@@ -817,6 +786,34 @@
}
/**
+ * Get the last modification timestamp of the message blob for this
module in a given language.
+ *
+ * @deprecated since 1.27 Superseded by getVersionHash()
+ * @param string $lang Language code
+ * @return int UNIX timestamp
+ */
+ public function getMsgBlobMtime( $lang ) {
+ if ( !$this->getMessages() ) {
+ return 1;
+ }
+ // Dummy that is > 1
+ return 2;
+ }
+
+ /**
+ * Set in-object cache for message blob time. Obsolete.
+ *
+ * Previously used to allow fetching of message blob store mtimes in
batches and
+ * then prepopulate the in-process. See
ResourceLoader::preloadModuleInfo().
+ *
+ * @param string $lang Language code
+ * @param int $mtime UNIX timestamp
+ */
+ public function setMsgBlobMtime( $lang, $mtime ) {
+ // No-op
+ }
+
+ /**
* Check whether this module is known to be empty. If a child class
* has an easy and cheap way to determine that this module is
* definitely going to be empty, it should override this method to
diff --git a/maintenance/cleanupRemovedModules.php
b/maintenance/cleanupRemovedModules.php
index ae05930..c22f7aa 100644
--- a/maintenance/cleanupRemovedModules.php
+++ b/maintenance/cleanupRemovedModules.php
@@ -59,18 +59,9 @@
} while ( $numRows > 0 );
$this->output( "done\n" );
- $this->output( "Cleaning up msg_resource table...\n" );
- $i = 1;
-
- $mrRes = $dbw->tableName( 'msg_resource' );
- do {
- $where = $moduleList ? "mr_resource NOT IN
($moduleList)" : '1=1';
- $dbw->query( "DELETE FROM $mrRes WHERE $where LIMIT
$limit", __METHOD__ );
- $numRows = $dbw->affectedRows();
- $this->output( "Batch $i: $numRows rows\n" );
- $i++;
- wfWaitForSlaves();
- } while ( $numRows > 0 );
+ $this->output( "Purging unused msg_resource table...\n" );
+ $mbs = $rl->getMessageBlobStore();
+ $mbs->clear();
$this->output( "done\n" );
}
}
diff --git a/tests/phpunit/includes/OutputPageTest.php
b/tests/phpunit/includes/OutputPageTest.php
index 5f21e07..f5ef016 100644
--- a/tests/phpunit/includes/OutputPageTest.php
+++ b/tests/phpunit/includes/OutputPageTest.php
@@ -375,7 +375,6 @@
}
public function updateModule( $name, ResourceLoaderModule $module,
$lang ) {
- return;
}
public function updateMessage( $key ) {
--
To view, visit https://gerrit.wikimedia.org/r/252880
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I78bb09bcea384dd166e5c13ff9b2e882c70e4df9
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/core
Gerrit-Branch: master
Gerrit-Owner: Krinkle <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
