jenkins-bot has submitted this change and it was merged.
Change subject: DB store
......................................................................
DB store
Change-Id: Ifd0bf6ee8ce468b487fd996f9dd8914120662622
---
M WikiGrok.php
A WikiGrok.sql
M includes/Hooks.php
A includes/QuestionStore.php
A maintenance/cleanupOldData.php
5 files changed, 310 insertions(+), 31 deletions(-)
Approvals:
Kaldari: Looks good to me, approved
jenkins-bot: Verified
diff --git a/WikiGrok.php b/WikiGrok.php
index 6d90f78..0dc1b87 100644
--- a/WikiGrok.php
+++ b/WikiGrok.php
@@ -24,6 +24,7 @@
'LinkedPropsCampaign' => 'campaigns/LinkedPropsCampaign',
'QuestionCache' => 'QuestionCache',
'QuestionPicker' => 'QuestionPicker',
+ 'QuestionStore' => 'QuestionStore',
'SimpleCampaign' => 'campaigns/SimpleCampaign',
'WikibaseConnector' => 'WikibaseConnector',
);
@@ -34,9 +35,10 @@
$wgAPIModules['wikigrokresponse'] = 'WikiGrok\Api\ApiResponse';
-$wgHooks['ParserBeforeTidy'][] = 'WikiGrok\Hooks::onParserBeforeTidy';
$wgHooks['OutputPageParserOutput'][] =
'WikiGrok\Hooks::onOutputPageParserOutput';
$wgHooks['LinksUpdate'][] = 'WikiGrok\Hooks::onLinksUpdate';
+$wgHooks['LoadExtensionSchemaUpdates'][] =
'WikiGrok\Hooks::onLoadExtensionSchemaUpdates';
+$wgHooks['ArticleDeleteComplete'][] =
'WikiGrok\Hooks::onArticleDeleteComplete';
$wgHooks['UnitTestsList'][] = function ( &$files ) {
$files[] = __DIR__ . '/tests/phpunit/api/ApiResponseTest.php';
};
@@ -67,6 +69,16 @@
// ),
// 'numLinks' => 10,
// ),
+// 'author' => array(
+// 'type' => 'Simple',
+// 'property' => 'P4'/* occupation */,
+// 'ifAll' => array(
+// 'P7'/* instance of */ => 'Q10'/* human */,
+// 'P4'/* occupation */ => 'Q4'/* writer */,
+// ),
+// 'ifNotAny' => array( 'P7'/* instance of */ => 'Q11'/* author */
),
+// 'suggestions' => array( 'Q11'/* author */ ),
+// ),
);
/**
diff --git a/WikiGrok.sql b/WikiGrok.sql
new file mode 100644
index 0000000..a36619d
--- /dev/null
+++ b/WikiGrok.sql
@@ -0,0 +1,42 @@
+-- Stores WikiGrok questions grouped by campaigns
+CREATE TABLE /*_*/wikigrok_questions (
+ -- Key to page_id
+ wgq_page int unsigned NOT NULL,
+ -- Campaign name, length matches page_props.pp_propname
+ wgq_campaign varchar(60) binary NOT NULL,
+ -- Serialization format version
+ wgq_version smallint default 1 NOT NULL,
+ -- Random number
+ wgq_random float NOT NULL,
+ -- Serialized campaign data
+ wgq_data blob NOT NULL,
+
+ -- Sample queries:
+ -- * SELECT ... FROM wikigrok_questions WHERE wgq_page = ... AND wgq_version
= ...
+ -- * INSERT IGNORE INTO wikigrok_questions VALUES (...)
+ -- * DELETE FROM wikigrok_questions WHERE wgq_page = ...
+ -- * DELETE FROM wikigrok_questions WHERE wgq_page = ... AND wgq_version <>
...
+ PRIMARY KEY(wgq_page, wgq_version, wgq_campaign)
+)/*$wgDBTableOptions*/;
+
+-- For cleaning up old campaigns
+-- Sample queries:
+-- * DELETE FROM wikigrok_questions WHERE wgq_campaign = ...
+-- * SELECT wgq_page FROM wikigrok_questions WHERE wgq_campaign = ... GROUP BY
wgq_page
+CREATE INDEX /*i*/wgq_campaign_page ON /*_*/wikigrok_questions(wgq_campaign,
wgq_page);
+
+-- For cleaning up old versions
+-- Sample queries:
+-- * DELETE FROM wikigrok_questions WHERE wgq_version <> ...
+-- * SELECT wgq_page FROM wikigrok_questions WHERE wgq_version = ... GROUP BY
wgq_page
+CREATE INDEX /*i*/wgq_version_page ON /*_*/wikigrok_questions(wgq_version,
wgq_page);
+
+-- For picking a random page
+-- Sample query:
+-- * SELECT ... FROM wikigrok_questions WHERE wgq_version = ... AND wgq_random
> ... ORDER BY wgq_random
+CREATE INDEX /*i*/wgq_version_random ON /*_*/wikigrok_questions(wgq_version,
wgq_random);
+
+-- For picking a random page within a campaign
+-- Sample query:
+-- * SELECT ... FROM wikigrok_questions WHERE wgq_version = ... AND
wgq_campaign = ... AND wgq_random > ... ORDER BY wgq_random
+CREATE INDEX /*i*/wgq_version_campaign_random ON
/*_*/wikigrok_questions(wgq_version, wgq_campaign, wgq_random);
diff --git a/includes/Hooks.php b/includes/Hooks.php
index 6b6c8af..5110979 100644
--- a/includes/Hooks.php
+++ b/includes/Hooks.php
@@ -6,31 +6,6 @@
* Hook handlers class
*/
class Hooks {
- /**
- * ParserBeforeTidy hook handler
- * Attaches suggestions to page properties
- * @see https://www.mediawiki.org/wiki/Manual:Hooks/ParserBeforeTidy
- *
- * @param \Parser $parser
- * @return bool
- */
- public static function onParserBeforeTidy( \Parser &$parser ) {
- $profileSection = new \ProfileSection( __METHOD__ );
-
- $parserOutput = $parser->getOutput();
- $campaigns = new QuestionPicker( self::getConfig() );
- try {
- $data = $campaigns->getSlowQuestions( $parserOutput );
- if ( $data ) {
- $value = serialize( $data );
- $parserOutput->setProperty(
QuestionPicker::CACHE_KEY, $value );
- }
- } catch ( \Exception $ex ) {
- wfDebugLog( 'mobile', __METHOD__ . "():
{$ex->getMessage()}" );
- }
-
- return true;
- }
/**
* OutputPageParserOutput hook handler
@@ -44,24 +19,26 @@
public static function onOutputPageParserOutput( \OutputPage $out,
\ParserOutput $parserOutput ) {
$profileSection = new \ProfileSection( __METHOD__ );
+ $item = $parserOutput->getProperty( 'wikibase_item' );
if ( !\MobileContext::singleton()->shouldDisplayMobileView()
|| $out->getUser()->isAnon()
+ || !$item
) {
return true;
}
try {
- $prop = $parserOutput->getProperty(
QuestionPicker::CACHE_KEY );
- $item = $parserOutput->getProperty( 'wikibase_item' );
$config = self::getConfig();
$campaigns = new QuestionPicker( $config );
+ $store = new QuestionStore();
if ( $config->get( 'WikiGrokDebug' ) ) {
$item = $out->getRequest()->getText(
'wikidataid', $item );
}
- $list = self::filterCampaigns(
- unserialize( $prop ),
+ $list =
+ self::filterCampaigns(
+ $store->load( $out->getTitle() ),
$config->get( 'WikiGrokSlowCampaigns' )
)
+ self::filterCampaigns(
@@ -80,7 +57,8 @@
/**
* LinksUpdate hook handler
- * Purges cache (currently only fast campaigns) for the page affected
+ * 1) Gets questions from slow campaigns and saves to DB
+ * 2) Purges fast campaigns cache for the page affected
* @see https://www.mediawiki.org/wiki/Manual:Hooks/LinksUpdate
*
* @param \LinksUpdate $lu
@@ -89,6 +67,16 @@
public static function onLinksUpdate( \LinksUpdate $lu ) {
$profileSection = new \ProfileSection( __METHOD__ );
+ $parserOutput = $lu->getParserOutput();
+ $campaigns = new QuestionPicker( self::getConfig() );
+ $data = array();
+ try {
+ $data = $campaigns->getSlowQuestions( $parserOutput );
+ } catch ( \Exception $ex ) {
+ wfDebugLog( 'mobile', __METHOD__ . "():
{$ex->getMessage()}" );
+ }
+ $store = new QuestionStore();
+ $store->store( $lu->getTitle(), $data );
$cache = new QuestionCache( self::getConfig() );
$cache->purge( $lu->getTitle()->getArticleID() );
@@ -96,6 +84,34 @@
}
/**
+ * ArticleDeleteComplete hook handler
+ * Deletes page information from the wikigrok_questions table
+ *
+ * @param \WikiPage $wikiPage unused
+ * @param \User $user unused
+ * @param string $reason unused
+ * @param int $id ID of page that was deleted
+ * @return bool
+ */
+ public static function onArticleDeleteComplete( $wikiPage, $user,
$reason, $id ) {
+ $store = new QuestionStore();
+ $store->delete( $id );
+ return true;
+ }
+
+ /**
+ * LoadExtensionSchemaUpdates hook handler
+ * @see
https://www.mediawiki.org/wiki/Manual:Hooks/LoadExtensionSchemaUpdates
+ *
+ * @param \DatabaseUpdater $updater
+ * @return bool
+ */
+ public static function onLoadExtensionSchemaUpdates( \DatabaseUpdater
$updater ) {
+ $updater->addExtensionTable( 'wikigrok_questions', dirname(
__DIR__ ) . '/WikiGrok.sql' );
+ return true;
+ }
+
+ /**
* Check cached campaigns in case there are campaigns disabled by now
*
* @param $prop
diff --git a/includes/QuestionStore.php b/includes/QuestionStore.php
new file mode 100644
index 0000000..cb82b7a
--- /dev/null
+++ b/includes/QuestionStore.php
@@ -0,0 +1,141 @@
+<?php
+
+namespace WikiGrok;
+
+use ProfileSection;
+use Title;
+
+/**
+ * Stores campaign questions in a permanent storage
+ */
+class QuestionStore {
+ const FORMAT_VERSION = 1;
+
+ public function store( Title $title, array $campaigns ) {
+ $profileSection = new ProfileSection( __METHOD__ );
+
+ $pageId = $title->getArticleID( Title::GAID_FOR_UPDATE );
+ if ( !$pageId ) {
+ return;
+ }
+
+ $dbw = wfGetDB( DB_MASTER );
+ $rows = array();
+ foreach ( $campaigns as $name => $data ) {
+ $rows[] = array(
+ 'wgq_page' => $pageId,
+ 'wgq_campaign' => $name,
+ 'wgq_version' => self::FORMAT_VERSION,
+ 'wgq_random' => mt_rand() / mt_getrandmax(),
+ 'wgq_data' => serialize( $data ),
+ );
+ }
+ if ( $rows ) {
+ $dbw->insert( 'wikigrok_questions', $rows, __METHOD__,
array( 'IGNORE' ) );
+ }
+
+ $conds = array( 'wgq_page' => $pageId );
+ // Kill old versions. Uncomment when there will be any
+ /*
+ $dbw->delete( 'wikigrok_questions',
+ $conds + array( 'wgq_version <> ' .
self::FORMAT_VERSION ),
+ __METHOD__
+ );*/
+ if ( $campaigns ) {
+ $names = self::campaignList( array_keys( $campaigns ) );
+ $conds[] = "wgq_campaign NOT IN($names)";
+ }
+ // Delete campaigns not in the new list
+ $dbw->delete( 'wikigrok_questions', $conds, __METHOD__ );
+ }
+
+ /**
+ * Loads campaign questions for the given title
+ *
+ * @param Title $title
+ * @return array Array of 'campaign name' => array( capaign data ),
same as returned by
+ * QuestionPicker::getQuestions()
+ */
+ public function load( Title $title ) {
+ $profileSection = new ProfileSection( __METHOD__ );
+
+ $dbr = wfGetDB( DB_SLAVE );
+ $res = $dbr->select( 'wikigrok_questions',
+ array( 'wgq_campaign', 'wgq_data' ),
+ array( 'wgq_page' => $title->getArticleID(),
'wgq_version' => self::FORMAT_VERSION ),
+ __METHOD__
+ );
+
+ $campaigns = array();
+ foreach ( $res as $row ) {
+ $campaigns[$row->wgq_campaign] = unserialize(
$row->wgq_data );
+ }
+ return $campaigns;
+ }
+
+ /**
+ * Returns a random page eligible for a given campaign
+ *
+ * @param string $campaign Campaign name. If null, a page for any
campaign will be returned
+ * @return Title|null
+ */
+ public function getRandomPage( $campaign = null ) {
+ $profileSection = new ProfileSection( __METHOD__ );
+
+ $dbr = wfGetDB( DB_SLAVE );
+ $conds = array(
+ 'wgq_version' => self::FORMAT_VERSION,
+ 'wgq_random > ' . ( mt_rand() / mt_getrandmax() ),
+ 'page_id = wgq_page'
+ );
+ if ( $campaign ) {
+ $conds['wgq_campaign'] = $campaign;
+ }
+ $table = $dbr->tableName( 'page' );
+ $res = $dbr->select(
+ array( 'wikigrok_questions', 'page' ),
+ array( 'wgq_page', "$table.*" ),
+ $conds,
+ __METHOD__,
+ array(
+ 'LIMIT' => 1,
+ 'ORDER BY' => 'wgq_random',
+ )
+ );
+ foreach ( $res as $row ) {
+ return Title::newFromRow( $row );
+ }
+ return null;
+ }
+
+ /**
+ * Deletes all data stored for a page
+ *
+ * @param $pageId
+ */
+ public function delete( $pageId ) {
+ $profileSection = new ProfileSection( __METHOD__ );
+
+ $dbw = wfGetDB( DB_MASTER );
+ $dbw->delete( 'wikigrok_questions', array( 'wgq_page' =>
$pageId ), __METHOD__ );
+ }
+
+ /**
+ * Returns a list of campaigns escaped and concatenated into a list
suitable for WHERE ... IN (...)
+ *
+ * @param array $campaigns
+ * @return string
+ */
+ public static function campaignList( array $campaigns ) {
+ $dbr = wfGetDB( DB_SLAVE );
+ return implode(
+ ', ',
+ array_map(
+ function( $name ) use( $dbr ) {
+ return $dbr->addQuotes( $name );
+ },
+ $campaigns
+ )
+ );
+ }
+}
diff --git a/maintenance/cleanupOldData.php b/maintenance/cleanupOldData.php
new file mode 100644
index 0000000..182a295
--- /dev/null
+++ b/maintenance/cleanupOldData.php
@@ -0,0 +1,68 @@
+<?php
+
+namespace WikiGrok;
+use Maintenance;
+
+$IP = getenv( 'MW_INSTALL_PATH' );
+if ( $IP === false ) {
+ $IP = dirname( __FILE__ ) . '/../..';
+}
+require_once( "$IP/maintenance/Maintenance.php" );
+
+class CleanupOldData extends Maintenance {
+
+ public function __construct() {
+ $this->mDescription = 'Cleans up database from old WikiGrok
data';
+ $this->addOption( 'old-versions', 'Removes data for previous
storage format versions' );
+ $this->addOption( 'old-campaigns', 'Removes data for
nonexistent campaigns' );
+ $this->mBatchSize = 500;
+ }
+ public function execute() {
+ $conds = array();
+ if ( $this->hasOption( 'old-versions' ) ) {
+ $conds = 'wgq_version <> ' .
QuestionStore::FORMAT_VERSION;
+ $this->output( "Deleting data for old versions...\n" );
+ $this->performDeletions( $conds );
+ }
+ if ( $this->hasOption( 'old-campaigns' ) ) {
+ $config =
\ConfigFactory::getDefaultInstance()->makeConfig( 'wikigrok' );
+ $campaigns = array_keys( $config->get(
'WikiGrokSlowCampaigns' ) );
+ if ( $campaigns ) {
+ $list = QuestionStore::campaignList(
array_keys( $campaigns ) );
+ $conds = "wgq_campaign NOT IN ($list)";
+ $this->output( "Deleting data for old
campaigns...\n" );
+ } else {
+ $conds = '1';
+ $this->output( "No campaigns configured,
deleting all data...\n" );
+ }
+ $this->performDeletions( $conds );
+ }
+ if ( !$conds ) {
+ $this->error( "Error: please specify what kind of data
to clean up!\n", 1 );
+ }
+ }
+
+ private function performDeletions( $conds ) {
+ $dbw = $this->getDB( DB_MASTER );
+ // Sigh, no abstraction for DELETE ... LIMIT ...
+ if ( $dbw->getType() != 'mysql' && $dbw->getType() != 'sqlite'
) {
+ $this->error( 'Only MySQL and SQLite are supported', 1
);
+ }
+ $count = 0;
+ $table = $dbw->tableName( 'wikigrok_questions' );
+ $func = __METHOD__;
+ $sql = "DELETE /*{$func}*/ FROM {$table} WHERE {$conds} LIMIT
{$this->mBatchSize}";
+ do {
+ $dbw->query( $sql, __METHOD__ );
+ $rows = $dbw->affectedRows();
+ $count += $rows;
+ if ( $rows ) {
+ echo " $count\n";
+ wfWaitForSlaves();
+ }
+ } while ( $rows );
+ }
+}
+
+$maintClass = 'WikiGrok\CleanupOldData';
+require_once RUN_MAINTENANCE_IF_MAIN;
--
To view, visit https://gerrit.wikimedia.org/r/170263
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ifd0bf6ee8ce468b487fd996f9dd8914120662622
Gerrit-PatchSet: 7
Gerrit-Project: mediawiki/extensions/WikiGrok
Gerrit-Branch: master
Gerrit-Owner: MaxSem <[email protected]>
Gerrit-Reviewer: Jdlrobson <[email protected]>
Gerrit-Reviewer: Kaldari <[email protected]>
Gerrit-Reviewer: MaxSem <[email protected]>
Gerrit-Reviewer: Phuedx <[email protected]>
Gerrit-Reviewer: Springle <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
