jenkins-bot has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/392469 )
Change subject: Add API action for dumping cirrus articles
......................................................................
Add API action for dumping cirrus articles
This is particularly convenient for the browser tests to use, so they
can ping the api to see if an article that it created/updated is now
in cirrussearch.
Change-Id: I5dbd02592eebb166362c7cb9dabcd2b93bae66c5
---
M CirrusSearch.php
M autoload.php
M i18n/en.json
M i18n/qqq.json
R includes/Api/ApiTrait.php
M includes/Api/ConfigDump.php
M includes/Api/FreezeWritesToCluster.php
M includes/Api/MappingDump.php
A includes/Api/QueryCirrusDoc.php
M includes/Api/SettingsDump.php
M includes/Api/SuggestIndex.php
M tests/integration/features/step_definitions/page_step_helpers.js
12 files changed, 157 insertions(+), 9 deletions(-)
Approvals:
Smalyshev: Looks good to me, approved
Cindy-the-browser-test-bot: Looks good to me, but someone else must approve
jenkins-bot: Verified
diff --git a/CirrusSearch.php b/CirrusSearch.php
index bf53382..0bd827c 100644
--- a/CirrusSearch.php
+++ b/CirrusSearch.php
@@ -1336,6 +1336,7 @@
$wgAPIModules['cirrus-config-dump'] = 'CirrusSearch\Api\ConfigDump';
$wgAPIModules['cirrus-mapping-dump'] = 'CirrusSearch\Api\MappingDump';
$wgAPIModules['cirrus-settings-dump'] = 'CirrusSearch\Api\SettingsDump';
+$wgAPIPropModules['cirrusdoc'] = 'CirrusSearch\Api\QueryCirrusDoc';
/**
* Configs
diff --git a/autoload.php b/autoload.php
index 1cb17de..f098fcc 100644
--- a/autoload.php
+++ b/autoload.php
@@ -5,10 +5,11 @@
$wgAutoloadClasses += [
'CirrusSearch' => __DIR__ . '/includes/CirrusSearch.php',
- 'CirrusSearch\\Api\\ApiBase' => __DIR__ . '/includes/Api/ApiBase.php',
+ 'CirrusSearch\\Api\\ApiTrait' => __DIR__ . '/includes/Api/ApiTrait.php',
'CirrusSearch\\Api\\ConfigDump' => __DIR__ .
'/includes/Api/ConfigDump.php',
'CirrusSearch\\Api\\FreezeWritesToCluster' => __DIR__ .
'/includes/Api/FreezeWritesToCluster.php',
'CirrusSearch\\Api\\MappingDump' => __DIR__ .
'/includes/Api/MappingDump.php',
+ 'CirrusSearch\\Api\\QueryCirrusDoc' => __DIR__ .
'/includes/Api/QueryCirrusDoc.php',
'CirrusSearch\\Api\\SettingsDump' => __DIR__ .
'/includes/Api/SettingsDump.php',
'CirrusSearch\\Api\\SuggestIndex' => __DIR__ .
'/includes/Api/SuggestIndex.php',
'CirrusSearch\\BaseInterwikiResolver' => __DIR__ .
'/includes/BaseInterwikiResolver.php',
diff --git a/i18n/en.json b/i18n/en.json
index ab44497..428c84c 100644
--- a/i18n/en.json
+++ b/i18n/en.json
@@ -23,6 +23,9 @@
"apihelp-cirrus-settings-dump-description": "Dump of CirrusSearch
settings for this wiki.",
"apihelp-cirrus-settings-dump-summary": "Dump of CirrusSearch settings
for this wiki.",
"apihelp-cirrus-settings-dump-example": "Get a dump of CirrusSearch
settings for this wiki.",
+ "apihelp-query+cirrusdoc-description": "Dump of a CirrusSearch article
document",
+ "apihelp-query+cirrusdoc-summary": "Dump of a CirrusSearch article
document",
+ "apihelp-query+cirrusdoc-example": "Get a dump of a single CirrusSearch
article.",
"apierror-cirrus-requesttoolong": "Prefix search request was longer
than the maximum allowed length. ($1 > $2)",
"cirrussearch-give-feedback": "Give us your feedback",
"cirrussearch-morelikethis-settings": " #<!-- leave this line exactly
as it is --> <pre>\n# This message lets you configure the settings of the
\"more like this\" feature.\n# Changes to this take effect immediately.\n# The
syntax is as follows:\n# * Everything from a \"#\" character to the end of
the line is a comment.\n# * Every non-blank line is the setting name followed
by a \":\" character followed by the setting value\n# The settings are:\n# *
min_doc_freq (integer): Minimum number of documents (per shard) that need a
term for it to be considered.\n# * max_doc_freq (integer): Maximum number of
documents (per shard) that have a term for it to be considered.\n#
High frequency terms are generally \"stop words\".\n# * max_query_terms
(integer): Maximum number of terms to be considered. This value is limited to
$wgCirrusSearchMoreLikeThisMaxQueryTermsLimit (100).\n# * min_term_freq
(integer): Minimum number of times the term appears in the input to doc to be
considered. For small fields (title) this value should be 1.\n# *
minimum_should_match (percentage -100% to 100%, or integer number of terms):
The percentage of terms to match on. Defaults to 30%.\n# * min_word_len
(integer): Minimal length of a term to be considered. Defaults to 0.\n# *
max_word_len (integer): The maximum word length above which words will be
ignored. Defaults to unbounded (0).\n# * fields (comma separated list of
values): These are the fields to use. Allowed fields are title, text,
auxiliary_text, opening_text, headings.\n# Examples of good lines:\n#
min_doc_freq:2\n# max_doc_freq:20000\n# max_query_terms:25\n#
min_term_freq:2\n# minimum_should_match:30%\n# min_word_len:2\n#
max_word_len:40\n# fields:text,opening_text\n# </pre> <!-- leave this line
exactly as it is -->",
diff --git a/i18n/qqq.json b/i18n/qqq.json
index 486768c..43b3492 100644
--- a/i18n/qqq.json
+++ b/i18n/qqq.json
@@ -33,6 +33,9 @@
"apihelp-cirrus-settings-dump-description":
"{{doc-apihelp-description|cirrus-settings-dump}}",
"apihelp-cirrus-settings-dump-summary":
"{{doc-apihelp-summary|cirrus-settings-dump}}",
"apihelp-cirrus-settings-dump-example":
"{{doc-apihelp-example|cirrus-settings-dump}}",
+ "apihelp-query+cirrusdoc-description":
"{{doc-apihelp-description|query+cirrusdoc}}",
+ "apihelp-query+cirrusdoc-summary":
"{{doc-apihelp-summary|query+cirrusdoc}}",
+ "apihelp-query+cirrusdoc-example":
"{{doc-apihelp-example|query+cirrusdoc}}",
"apierror-cirrus-requesttoolong": "{{doc-apierror}}\n\nParameters:\n*
$1 - Submitted request length\n* $2 - Maximum allowed request length",
"cirrussearch-give-feedback": "Used as text for an feedback link shown
at the end of Special:Search result
([[mw:Extension:CirrusSearch|$wgCirrusSearchFeedbackLink]])",
"cirrussearch-morelikethis-settings": "Settings for the More Like This
query.\n\n\"More Like This\" is the English name of the feature. The feature is
described at [[:mw:Help:CirrusSearch#Special prefixes]]. The prefix
\"morelike\" cannot be translated anywhere, but the full name of the feature
\"More Like This\" can be translated.\n\nDon't translate technical names like
min_doc_freq, max_query_terms, true|false, field names title, text,
auxiliary_text, opening_text, headings, all etc.\n\nFor a definition of
\"stopwords\" see [[:w:en:Stop words|Stop words in Wikipedia]].",
diff --git a/includes/Api/ApiBase.php b/includes/Api/ApiTrait.php
similarity index 90%
rename from includes/Api/ApiBase.php
rename to includes/Api/ApiTrait.php
index 227f834..ec53232 100644
--- a/includes/Api/ApiBase.php
+++ b/includes/Api/ApiTrait.php
@@ -2,12 +2,11 @@
namespace CirrusSearch\Api;
-use ApiBase as CoreApiBase;
use CirrusSearch\Connection;
use CirrusSearch\SearchConfig;
use MediaWiki\MediaWikiServices;
-abstract class ApiBase extends CoreApiBase {
+trait ApiTrait {
/** @var Connection */
private $connection;
/** @var SearchConfig */
diff --git a/includes/Api/ConfigDump.php b/includes/Api/ConfigDump.php
index c79fb84..e3d3c1f 100644
--- a/includes/Api/ConfigDump.php
+++ b/includes/Api/ConfigDump.php
@@ -20,7 +20,9 @@
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*/
-class ConfigDump extends ApiBase {
+class ConfigDump extends \ApiBase {
+ use ApiTrait;
+
public static $WHITE_LIST = [
'CirrusSearchServers',
'CirrusSearchConnectionAttempts',
diff --git a/includes/Api/FreezeWritesToCluster.php
b/includes/Api/FreezeWritesToCluster.php
index d157353..d0c9f5f 100644
--- a/includes/Api/FreezeWritesToCluster.php
+++ b/includes/Api/FreezeWritesToCluster.php
@@ -23,7 +23,9 @@
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*/
-class FreezeWritesToCluster extends ApiBase {
+class FreezeWritesToCluster extends \ApiBase {
+ use ApiTrait;
+
public function execute() {
$sender = new DataSender( $this->getCirrusConnection(),
$this->getSearchConfig() );
diff --git a/includes/Api/MappingDump.php b/includes/Api/MappingDump.php
index fc180d4..7b80822 100644
--- a/includes/Api/MappingDump.php
+++ b/includes/Api/MappingDump.php
@@ -23,7 +23,9 @@
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*/
-class MappingDump extends ApiBase {
+class MappingDump extends \ApiBase {
+ use ApiTrait;
+
public function execute() {
$conn = $this->getCirrusConnection();
$indexPrefix = $this->getSearchConfig()->get(
SearchConfig::INDEX_BASE_NAME );
diff --git a/includes/Api/QueryCirrusDoc.php b/includes/Api/QueryCirrusDoc.php
new file mode 100644
index 0000000..1074f7c
--- /dev/null
+++ b/includes/Api/QueryCirrusDoc.php
@@ -0,0 +1,93 @@
+<?php
+
+namespace CirrusSearch\Api;
+
+use CirrusSearch\Searcher;
+use CirrusSearch\Updater;
+
+/**
+ * Dump stored CirrusSearch document for page.
+ *
+ * This was primarily written for the integration tests, but may be useful
+ * elsewhere. This is functionally similar to web action=cirrusdump but
+ * available and discoverable over the API.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ */
+class QueryCirrusDoc extends \ApiQueryBase {
+ use ApiTrait;
+
+ public function __construct( \ApiQuery $query, $moduleName ) {
+ parent::__construct( $query, $moduleName, 'cd' );
+ }
+
+ public function execute() {
+ $conn = $this->getCirrusConnection();
+ $config = $this->getSearchConfig();
+ $updater = new Updater( $conn, $config );
+ $searcher = new Searcher( $conn, 0, 0, $config, [],
$this->getUser() );
+ $result = [];
+ foreach ( $this->getPageSet()->getGoodTitles() as $origPageId
=> $title ) {
+ list( $page, $redirects ) = $updater->traceRedirects(
$title );
+
+ $result = [];
+ if ( $page ) {
+ $docId = $config->makeId( $page->getId() );
+ // could be optimized by implementing multi-get
but not
+ // expecting much usage except debugging/tests.
+ $esSources = $searcher->get( [ $docId ], true );
+ if ( $esSources->isOK() ) {
+ foreach ( $esSources->getValue() as $i
=> $esSource ) {
+ $result[] = [
+ 'index' =>
$esSource->getIndex(),
+ 'type' =>
$esSource->getType(),
+ 'id' =>
$esSource->getId(),
+ 'version' =>
$esSource->getVersion(),
+ 'source' =>
$esSource->getData(),
+ ];
+ }
+ }
+ }
+ $this->getResult()->addValue(
+ [ 'query', 'pages', $origPageId ],
+ 'cirrusdoc', $result
+ );
+ }
+ }
+
+ public function getAllowedParams() {
+ return [];
+ }
+
+ /**
+ * @deprecated since MediaWiki core 1.25
+ */
+ public function getDescription() {
+ return 'Dump stored CirrusSearch document for page.';
+ }
+
+ /**
+ * @see ApiBase::getExamplesMessages
+ * @return array
+ */
+ protected function getExamplesMessages() {
+ return [
+ 'action=query&prop=cirrusdoc&titles=Main_Page' =>
+ 'apihelp-query+cirrusdoc-example'
+ ];
+ }
+
+}
diff --git a/includes/Api/SettingsDump.php b/includes/Api/SettingsDump.php
index c18baa6..2462896 100644
--- a/includes/Api/SettingsDump.php
+++ b/includes/Api/SettingsDump.php
@@ -22,7 +22,9 @@
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*/
-class SettingsDump extends ApiBase {
+class SettingsDump extends \ApiBase {
+ use ApiTrait;
+
public function execute() {
$conn = $this->getCirrusConnection();
$indexPrefix = $this->getSearchConfig()->get(
SearchConfig::INDEX_BASE_NAME );
diff --git a/includes/Api/SuggestIndex.php b/includes/Api/SuggestIndex.php
index 9484d5b..764b2d1 100644
--- a/includes/Api/SuggestIndex.php
+++ b/includes/Api/SuggestIndex.php
@@ -20,7 +20,8 @@
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*/
-class SuggestIndex extends ApiBase {
+class SuggestIndex extends \ApiBase {
+ use ApiTrait;
public function execute() {
// FIXME: This is horrible, no good, very bad hack. Only for
testing,
diff --git a/tests/integration/features/step_definitions/page_step_helpers.js
b/tests/integration/features/step_definitions/page_step_helpers.js
index a67761e..e250504 100644
--- a/tests/integration/features/step_definitions/page_step_helpers.js
+++ b/tests/integration/features/step_definitions/page_step_helpers.js
@@ -12,7 +12,8 @@
const expect = require( 'chai' ).expect,
fs = require( 'fs' ),
- path = require( 'path' );
+ path = require( 'path' ),
+ Promise = require( 'bluebird' ); // jshint ignore:line
class StepHelpers {
constructor( world, wiki ) {
@@ -110,6 +111,44 @@
( response ) => this.world.setApiResponse( response ),
( error ) => this.world.setApiError( error ) );
}
+
+ waitForOperation( operation, title ) {
+ return Promise.coroutine( function* () {
+ let expect = operation === 'delete' ? false : true;
+ let exists;
+ do {
+ exists = yield this.checkExists( title );
+ } while ( expect !== exists );
+ } ).call( this );
+ }
+
+ checkExists( title ) {
+ return Promise.coroutine( function* () {
+ let client = yield this.apiPromise;
+ let response = yield client.request( {
+ action: 'query',
+ prop: 'cirrusdoc',
+ titles: title,
+ format: 'json',
+ formatversion: 2
+ } );
+ if ( response.query.normalized ) {
+ for ( let norm of response.query.normalized ) {
+ if ( norm.from === title ) {
+ title = norm.to;
+ break;
+ }
+ }
+ }
+ for ( let page of response.query.pages ) {
+ if ( page.title === title ) {
+ // without boolean cast we could return
undefined
+ return Boolean( page.cirrusdoc &&
page.cirrusdoc.length > 0 );
+ }
+ }
+ return false;
+ } ).call( this );
+ }
}
module.exports = StepHelpers;
--
To view, visit https://gerrit.wikimedia.org/r/392469
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I5dbd02592eebb166362c7cb9dabcd2b93bae66c5
Gerrit-PatchSet: 7
Gerrit-Project: mediawiki/extensions/CirrusSearch
Gerrit-Branch: master
Gerrit-Owner: EBernhardson <ebernhard...@wikimedia.org>
Gerrit-Reviewer: Cindy-the-browser-test-bot <bernhardsone...@gmail.com>
Gerrit-Reviewer: DCausse <dcau...@wikimedia.org>
Gerrit-Reviewer: EBernhardson <ebernhard...@wikimedia.org>
Gerrit-Reviewer: Gehel <guillaume.leder...@wikimedia.org>
Gerrit-Reviewer: Smalyshev <smalys...@wikimedia.org>
Gerrit-Reviewer: Tjones <tjo...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
