jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/392469 )
Change subject: Add API action for dumping cirrus articles ...................................................................... Add API action for dumping cirrus articles This is particularly convenient for the browser tests to use, so they can ping the api to see if an article that it created/updated is now in cirrussearch. Change-Id: I5dbd02592eebb166362c7cb9dabcd2b93bae66c5 --- M CirrusSearch.php M autoload.php M i18n/en.json M i18n/qqq.json R includes/Api/ApiTrait.php M includes/Api/ConfigDump.php M includes/Api/FreezeWritesToCluster.php M includes/Api/MappingDump.php A includes/Api/QueryCirrusDoc.php M includes/Api/SettingsDump.php M includes/Api/SuggestIndex.php M tests/integration/features/step_definitions/page_step_helpers.js 12 files changed, 157 insertions(+), 9 deletions(-) Approvals: Smalyshev: Looks good to me, approved Cindy-the-browser-test-bot: Looks good to me, but someone else must approve jenkins-bot: Verified diff --git a/CirrusSearch.php b/CirrusSearch.php index bf53382..0bd827c 100644 --- a/CirrusSearch.php +++ b/CirrusSearch.php @@ -1336,6 +1336,7 @@ $wgAPIModules['cirrus-config-dump'] = 'CirrusSearch\Api\ConfigDump'; $wgAPIModules['cirrus-mapping-dump'] = 'CirrusSearch\Api\MappingDump'; $wgAPIModules['cirrus-settings-dump'] = 'CirrusSearch\Api\SettingsDump'; +$wgAPIPropModules['cirrusdoc'] = 'CirrusSearch\Api\QueryCirrusDoc'; /** * Configs diff --git a/autoload.php b/autoload.php index 1cb17de..f098fcc 100644 --- a/autoload.php +++ b/autoload.php @@ -5,10 +5,11 @@ $wgAutoloadClasses += [ 'CirrusSearch' => __DIR__ . '/includes/CirrusSearch.php', - 'CirrusSearch\\Api\\ApiBase' => __DIR__ . '/includes/Api/ApiBase.php', + 'CirrusSearch\\Api\\ApiTrait' => __DIR__ . '/includes/Api/ApiTrait.php', 'CirrusSearch\\Api\\ConfigDump' => __DIR__ . '/includes/Api/ConfigDump.php', 'CirrusSearch\\Api\\FreezeWritesToCluster' => __DIR__ . '/includes/Api/FreezeWritesToCluster.php', 'CirrusSearch\\Api\\MappingDump' => __DIR__ . '/includes/Api/MappingDump.php', + 'CirrusSearch\\Api\\QueryCirrusDoc' => __DIR__ . '/includes/Api/QueryCirrusDoc.php', 'CirrusSearch\\Api\\SettingsDump' => __DIR__ . '/includes/Api/SettingsDump.php', 'CirrusSearch\\Api\\SuggestIndex' => __DIR__ . '/includes/Api/SuggestIndex.php', 'CirrusSearch\\BaseInterwikiResolver' => __DIR__ . '/includes/BaseInterwikiResolver.php', diff --git a/i18n/en.json b/i18n/en.json index ab44497..428c84c 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -23,6 +23,9 @@ "apihelp-cirrus-settings-dump-description": "Dump of CirrusSearch settings for this wiki.", "apihelp-cirrus-settings-dump-summary": "Dump of CirrusSearch settings for this wiki.", "apihelp-cirrus-settings-dump-example": "Get a dump of CirrusSearch settings for this wiki.", + "apihelp-query+cirrusdoc-description": "Dump of a CirrusSearch article document", + "apihelp-query+cirrusdoc-summary": "Dump of a CirrusSearch article document", + "apihelp-query+cirrusdoc-example": "Get a dump of a single CirrusSearch article.", "apierror-cirrus-requesttoolong": "Prefix search request was longer than the maximum allowed length. ($1 > $2)", "cirrussearch-give-feedback": "Give us your feedback", "cirrussearch-morelikethis-settings": " #<!-- leave this line exactly as it is --> <pre>\n# This message lets you configure the settings of the \"more like this\" feature.\n# Changes to this take effect immediately.\n# The syntax is as follows:\n# * Everything from a \"#\" character to the end of the line is a comment.\n# * Every non-blank line is the setting name followed by a \":\" character followed by the setting value\n# The settings are:\n# * min_doc_freq (integer): Minimum number of documents (per shard) that need a term for it to be considered.\n# * max_doc_freq (integer): Maximum number of documents (per shard) that have a term for it to be considered.\n# High frequency terms are generally \"stop words\".\n# * max_query_terms (integer): Maximum number of terms to be considered. This value is limited to $wgCirrusSearchMoreLikeThisMaxQueryTermsLimit (100).\n# * min_term_freq (integer): Minimum number of times the term appears in the input to doc to be considered. For small fields (title) this value should be 1.\n# * minimum_should_match (percentage -100% to 100%, or integer number of terms): The percentage of terms to match on. Defaults to 30%.\n# * min_word_len (integer): Minimal length of a term to be considered. Defaults to 0.\n# * max_word_len (integer): The maximum word length above which words will be ignored. Defaults to unbounded (0).\n# * fields (comma separated list of values): These are the fields to use. Allowed fields are title, text, auxiliary_text, opening_text, headings.\n# Examples of good lines:\n# min_doc_freq:2\n# max_doc_freq:20000\n# max_query_terms:25\n# min_term_freq:2\n# minimum_should_match:30%\n# min_word_len:2\n# max_word_len:40\n# fields:text,opening_text\n# </pre> <!-- leave this line exactly as it is -->", diff --git a/i18n/qqq.json b/i18n/qqq.json index 486768c..43b3492 100644 --- a/i18n/qqq.json +++ b/i18n/qqq.json @@ -33,6 +33,9 @@ "apihelp-cirrus-settings-dump-description": "{{doc-apihelp-description|cirrus-settings-dump}}", "apihelp-cirrus-settings-dump-summary": "{{doc-apihelp-summary|cirrus-settings-dump}}", "apihelp-cirrus-settings-dump-example": "{{doc-apihelp-example|cirrus-settings-dump}}", + "apihelp-query+cirrusdoc-description": "{{doc-apihelp-description|query+cirrusdoc}}", + "apihelp-query+cirrusdoc-summary": "{{doc-apihelp-summary|query+cirrusdoc}}", + "apihelp-query+cirrusdoc-example": "{{doc-apihelp-example|query+cirrusdoc}}", "apierror-cirrus-requesttoolong": "{{doc-apierror}}\n\nParameters:\n* $1 - Submitted request length\n* $2 - Maximum allowed request length", "cirrussearch-give-feedback": "Used as text for an feedback link shown at the end of Special:Search result ([[mw:Extension:CirrusSearch|$wgCirrusSearchFeedbackLink]])", "cirrussearch-morelikethis-settings": "Settings for the More Like This query.\n\n\"More Like This\" is the English name of the feature. The feature is described at [[:mw:Help:CirrusSearch#Special prefixes]]. The prefix \"morelike\" cannot be translated anywhere, but the full name of the feature \"More Like This\" can be translated.\n\nDon't translate technical names like min_doc_freq, max_query_terms, true|false, field names title, text, auxiliary_text, opening_text, headings, all etc.\n\nFor a definition of \"stopwords\" see [[:w:en:Stop words|Stop words in Wikipedia]].", diff --git a/includes/Api/ApiBase.php b/includes/Api/ApiTrait.php similarity index 90% rename from includes/Api/ApiBase.php rename to includes/Api/ApiTrait.php index 227f834..ec53232 100644 --- a/includes/Api/ApiBase.php +++ b/includes/Api/ApiTrait.php @@ -2,12 +2,11 @@ namespace CirrusSearch\Api; -use ApiBase as CoreApiBase; use CirrusSearch\Connection; use CirrusSearch\SearchConfig; use MediaWiki\MediaWikiServices; -abstract class ApiBase extends CoreApiBase { +trait ApiTrait { /** @var Connection */ private $connection; /** @var SearchConfig */ diff --git a/includes/Api/ConfigDump.php b/includes/Api/ConfigDump.php index c79fb84..e3d3c1f 100644 --- a/includes/Api/ConfigDump.php +++ b/includes/Api/ConfigDump.php @@ -20,7 +20,9 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * http://www.gnu.org/copyleft/gpl.html */ -class ConfigDump extends ApiBase { +class ConfigDump extends \ApiBase { + use ApiTrait; + public static $WHITE_LIST = [ 'CirrusSearchServers', 'CirrusSearchConnectionAttempts', diff --git a/includes/Api/FreezeWritesToCluster.php b/includes/Api/FreezeWritesToCluster.php index d157353..d0c9f5f 100644 --- a/includes/Api/FreezeWritesToCluster.php +++ b/includes/Api/FreezeWritesToCluster.php @@ -23,7 +23,9 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * http://www.gnu.org/copyleft/gpl.html */ -class FreezeWritesToCluster extends ApiBase { +class FreezeWritesToCluster extends \ApiBase { + use ApiTrait; + public function execute() { $sender = new DataSender( $this->getCirrusConnection(), $this->getSearchConfig() ); diff --git a/includes/Api/MappingDump.php b/includes/Api/MappingDump.php index fc180d4..7b80822 100644 --- a/includes/Api/MappingDump.php +++ b/includes/Api/MappingDump.php @@ -23,7 +23,9 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * http://www.gnu.org/copyleft/gpl.html */ -class MappingDump extends ApiBase { +class MappingDump extends \ApiBase { + use ApiTrait; + public function execute() { $conn = $this->getCirrusConnection(); $indexPrefix = $this->getSearchConfig()->get( SearchConfig::INDEX_BASE_NAME ); diff --git a/includes/Api/QueryCirrusDoc.php b/includes/Api/QueryCirrusDoc.php new file mode 100644 index 0000000..1074f7c --- /dev/null +++ b/includes/Api/QueryCirrusDoc.php @@ -0,0 +1,93 @@ +<?php + +namespace CirrusSearch\Api; + +use CirrusSearch\Searcher; +use CirrusSearch\Updater; + +/** + * Dump stored CirrusSearch document for page. + * + * This was primarily written for the integration tests, but may be useful + * elsewhere. This is functionally similar to web action=cirrusdump but + * available and discoverable over the API. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + */ +class QueryCirrusDoc extends \ApiQueryBase { + use ApiTrait; + + public function __construct( \ApiQuery $query, $moduleName ) { + parent::__construct( $query, $moduleName, 'cd' ); + } + + public function execute() { + $conn = $this->getCirrusConnection(); + $config = $this->getSearchConfig(); + $updater = new Updater( $conn, $config ); + $searcher = new Searcher( $conn, 0, 0, $config, [], $this->getUser() ); + $result = []; + foreach ( $this->getPageSet()->getGoodTitles() as $origPageId => $title ) { + list( $page, $redirects ) = $updater->traceRedirects( $title ); + + $result = []; + if ( $page ) { + $docId = $config->makeId( $page->getId() ); + // could be optimized by implementing multi-get but not + // expecting much usage except debugging/tests. + $esSources = $searcher->get( [ $docId ], true ); + if ( $esSources->isOK() ) { + foreach ( $esSources->getValue() as $i => $esSource ) { + $result[] = [ + 'index' => $esSource->getIndex(), + 'type' => $esSource->getType(), + 'id' => $esSource->getId(), + 'version' => $esSource->getVersion(), + 'source' => $esSource->getData(), + ]; + } + } + } + $this->getResult()->addValue( + [ 'query', 'pages', $origPageId ], + 'cirrusdoc', $result + ); + } + } + + public function getAllowedParams() { + return []; + } + + /** + * @deprecated since MediaWiki core 1.25 + */ + public function getDescription() { + return 'Dump stored CirrusSearch document for page.'; + } + + /** + * @see ApiBase::getExamplesMessages + * @return array + */ + protected function getExamplesMessages() { + return [ + 'action=query&prop=cirrusdoc&titles=Main_Page' => + 'apihelp-query+cirrusdoc-example' + ]; + } + +} diff --git a/includes/Api/SettingsDump.php b/includes/Api/SettingsDump.php index c18baa6..2462896 100644 --- a/includes/Api/SettingsDump.php +++ b/includes/Api/SettingsDump.php @@ -22,7 +22,9 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * http://www.gnu.org/copyleft/gpl.html */ -class SettingsDump extends ApiBase { +class SettingsDump extends \ApiBase { + use ApiTrait; + public function execute() { $conn = $this->getCirrusConnection(); $indexPrefix = $this->getSearchConfig()->get( SearchConfig::INDEX_BASE_NAME ); diff --git a/includes/Api/SuggestIndex.php b/includes/Api/SuggestIndex.php index 9484d5b..764b2d1 100644 --- a/includes/Api/SuggestIndex.php +++ b/includes/Api/SuggestIndex.php @@ -20,7 +20,8 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * http://www.gnu.org/copyleft/gpl.html */ -class SuggestIndex extends ApiBase { +class SuggestIndex extends \ApiBase { + use ApiTrait; public function execute() { // FIXME: This is horrible, no good, very bad hack. Only for testing, diff --git a/tests/integration/features/step_definitions/page_step_helpers.js b/tests/integration/features/step_definitions/page_step_helpers.js index a67761e..e250504 100644 --- a/tests/integration/features/step_definitions/page_step_helpers.js +++ b/tests/integration/features/step_definitions/page_step_helpers.js @@ -12,7 +12,8 @@ const expect = require( 'chai' ).expect, fs = require( 'fs' ), - path = require( 'path' ); + path = require( 'path' ), + Promise = require( 'bluebird' ); // jshint ignore:line class StepHelpers { constructor( world, wiki ) { @@ -110,6 +111,44 @@ ( response ) => this.world.setApiResponse( response ), ( error ) => this.world.setApiError( error ) ); } + + waitForOperation( operation, title ) { + return Promise.coroutine( function* () { + let expect = operation === 'delete' ? false : true; + let exists; + do { + exists = yield this.checkExists( title ); + } while ( expect !== exists ); + } ).call( this ); + } + + checkExists( title ) { + return Promise.coroutine( function* () { + let client = yield this.apiPromise; + let response = yield client.request( { + action: 'query', + prop: 'cirrusdoc', + titles: title, + format: 'json', + formatversion: 2 + } ); + if ( response.query.normalized ) { + for ( let norm of response.query.normalized ) { + if ( norm.from === title ) { + title = norm.to; + break; + } + } + } + for ( let page of response.query.pages ) { + if ( page.title === title ) { + // without boolean cast we could return undefined + return Boolean( page.cirrusdoc && page.cirrusdoc.length > 0 ); + } + } + return false; + } ).call( this ); + } } module.exports = StepHelpers; -- To view, visit https://gerrit.wikimedia.org/r/392469 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I5dbd02592eebb166362c7cb9dabcd2b93bae66c5 Gerrit-PatchSet: 7 Gerrit-Project: mediawiki/extensions/CirrusSearch Gerrit-Branch: master Gerrit-Owner: EBernhardson <ebernhard...@wikimedia.org> Gerrit-Reviewer: Cindy-the-browser-test-bot <bernhardsone...@gmail.com> Gerrit-Reviewer: DCausse <dcau...@wikimedia.org> Gerrit-Reviewer: EBernhardson <ebernhard...@wikimedia.org> Gerrit-Reviewer: Gehel <guillaume.leder...@wikimedia.org> Gerrit-Reviewer: Smalyshev <smalys...@wikimedia.org> Gerrit-Reviewer: Tjones <tjo...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits