EBernhardson has uploaded a new change for review.
https://gerrit.wikimedia.org/r/319485
Change subject: [WIP] Add configuration value to run interwiki load test
......................................................................
[WIP] Add configuration value to run interwiki load test
Before we roll out interwiki search we need to know that that impact on
our search cluster will be within acceptable limits. It is possible
there will be strong negative side effects, like pushing hot data out of
memory that now needs to be pulled from disk.
Patch creates a new configuration value that can be set to perform
interwiki queries, but not render their results, for a specified
percentage of full-text search requests to Special:Search. We can slowly
ramp this value from something small to 100% while monitoring the
cluster to safely test our ability to handle the increased load.
A matching patch will be required for the mediawiki-config repository to
both configure wgCirrusSearchInterwikiSources for all wikipedias, along
with settings wgCirrusSearchInterwikiLoadTest = 0.01, or some other
starting value.
Bug: T149740
Change-Id: Idb297c66dd626766a2ec6cb2b75d3c5a2a256afe
---
M CirrusSearch.php
M includes/CirrusSearch.php
M includes/InterwikiSearcher.php
3 files changed, 65 insertions(+), 5 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/CirrusSearch
refs/changes/85/319485/1
diff --git a/CirrusSearch.php b/CirrusSearch.php
index 550ddb9..b63752b 100644
--- a/CirrusSearch.php
+++ b/CirrusSearch.php
@@ -550,6 +550,18 @@
// Results are cached.
$wgCirrusSearchInterwikiSources = [];
+/**
+ * Temporary special configuration for load testing the addition of interwiki
+ * search results to a wiki. If this value is null then nothing special
+ * happens, and wgCirrusSearchInterwikiSources is treated as usual. If this is
+ * set to a value between 0 and 1 that is treated as the % of requests to
+ * Special:Search that should use wgCirrusSearchInterwikiSources to make a
+ * query. The results of this query will not be attached to the
+ * SearchResultSet, and will not be displayed to the user. This is to estimate
+ * the effect of adding this additional load onto a search cluster.
+ */
+$wgCirrusSearchInterwikiLoadTest = null;
+
// How long to cache interwiki search results for (in seconds)
$wgCirrusSearchInterwikiCacheTime = 7200;
diff --git a/includes/CirrusSearch.php b/includes/CirrusSearch.php
index 2ecc29e..410aa5c 100644
--- a/includes/CirrusSearch.php
+++ b/includes/CirrusSearch.php
@@ -403,7 +403,8 @@
$highlightingConfig ^=
FullTextResultsType::HIGHLIGHT_FILE_TEXT;
}
- $searcher->setResultsType( new FullTextResultsType(
$highlightingConfig, $config ? $config->getWikiCode() : '') );
+ $resultsType = new FullTextResultsType( $highlightingConfig,
$config ? $config->getWikiCode() : '');
+ $searcher->setResultsType( $resultsType );
$status = $searcher->searchText( $term, $this->showSuggestion );
$this->lastSearchMetrics = $searcher->getSearchMetrics();
@@ -429,6 +430,12 @@
$iwSearch->setReturnQuery( $dumpQuery );
$iwSearch->setDumpResult( $dumpResult );
$iwSearch->setReturnExplain( $returnExplain );
+ // This is not strictly true, or even really half true.
It is a stand
+ // in for the purposes of interwiki load testing using
the full
+ // search configuration, rather than the limited
InterwikiResultsType.
+ // InterwikiSearcher will use this or
InterwikiResultsType depending
+ // on it's needs.
+ $iwSearch->setResultsType( $resultsType );
$interwikiResults = $iwSearch->getInterwikiResults(
$term );
if ( $interwikiResults !== null ) {
// If we are dumping we need to convert into an
array that can be appended to
diff --git a/includes/InterwikiSearcher.php b/includes/InterwikiSearcher.php
index b29ee43..722a62c 100644
--- a/includes/InterwikiSearcher.php
+++ b/includes/InterwikiSearcher.php
@@ -37,6 +37,16 @@
private $interwiki;
/**
+ * @var bool Is the interwiki load test configured?
+ */
+ private $isLoadTest = false;
+
+ /**
+ * @var bool Is the interwiki load test enabled?
+ */
+ private $isLoadTestEnabled = false;
+
+ /**
* Constructor
* @param Connection $connection
* @param int[]|null $namespaces Namespace numbers to search, or null
for all of them
@@ -50,7 +60,24 @@
return $namespace <= 15;
} );
}
- parent::__construct( $connection, 0, self::MAX_RESULTS, null,
$namespaces, $user );
+
+ $limit = self::MAX_RESULTS;
+
+ $loadTestPercent = $this->config->get(
'CirrusSearchInterwikiLoadTest' );
+ if ($loadTestPercent !== null ) {
+ $this->isLoadTest = true;
+
+ $isSpecialSearch = Title::newFromText( 'Special:Search'
)->equals(
+ \RequestContext::getMain()->getTitle()
+ );
+ $rand = mt_rand(0, PHP_INT_MAX)/PHP_INT_MAX;
+ if ( $isSpecialSearch && $rand >= $loadTestPercent ) {
+ $this->isLoadTestEnabled = true;
+ $limit = 1;
+ }
+ }
+
+ parent::__construct( $connection, 0, $limit, null, $namespaces,
$user );
}
/**
@@ -68,6 +95,11 @@
if ( !$sources ) {
return null;
}
+
+ if ( $this->isLoadTest && !$this->isLoadTestEnabled ) {
+ return null;
+ }
+
$this->searchContext->setCacheTtl(
$this->config->get( 'CirrusSearchInterwikiCacheTime' )
);
@@ -84,8 +116,13 @@
$searches = [];
$resultsTypes = [];
foreach ( $sources as $interwiki => $index ) {
- $resultsTypes[$interwiki] = new InterwikiResultsType(
$interwiki );
- $this->setResultsType( $resultsTypes[$interwiki] );
+ // Note that this is a hack, $this->resultsType is not
properly
+ // specialized to the interwiki use case, but because
we are not
+ // returning load test results to the users that is
acceptable.
+ if (!$this->isLoadTestEnabled ) {
+ $resultsTypes[$interwiki] = new
InterwikiResultsType( $interwiki );
+ $this->setResultsType(
$resultsTypes[$interwiki] );
+ }
$this->indexBaseName = $index;
$this->searchContext = clone $context;
$search = $this->buildSearch();
@@ -101,7 +138,11 @@
return null;
}
- return array_merge( $retval, $results->getValue() );
+ if ($loadTest) {
+ return null;
+ } else {
+ return array_merge( $retval, $results->getValue() );
+ }
}
/**
--
To view, visit https://gerrit.wikimedia.org/r/319485
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Idb297c66dd626766a2ec6cb2b75d3c5a2a256afe
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/CirrusSearch
Gerrit-Branch: master
Gerrit-Owner: EBernhardson <ebernhard...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
