EBernhardson has uploaded a new change for review. https://gerrit.wikimedia.org/r/319485
Change subject: [WIP] Add configuration value to run interwiki load test ...................................................................... [WIP] Add configuration value to run interwiki load test Before we roll out interwiki search we need to know that that impact on our search cluster will be within acceptable limits. It is possible there will be strong negative side effects, like pushing hot data out of memory that now needs to be pulled from disk. Patch creates a new configuration value that can be set to perform interwiki queries, but not render their results, for a specified percentage of full-text search requests to Special:Search. We can slowly ramp this value from something small to 100% while monitoring the cluster to safely test our ability to handle the increased load. A matching patch will be required for the mediawiki-config repository to both configure wgCirrusSearchInterwikiSources for all wikipedias, along with settings wgCirrusSearchInterwikiLoadTest = 0.01, or some other starting value. Bug: T149740 Change-Id: Idb297c66dd626766a2ec6cb2b75d3c5a2a256afe --- M CirrusSearch.php M includes/CirrusSearch.php M includes/InterwikiSearcher.php 3 files changed, 65 insertions(+), 5 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/CirrusSearch refs/changes/85/319485/1 diff --git a/CirrusSearch.php b/CirrusSearch.php index 550ddb9..b63752b 100644 --- a/CirrusSearch.php +++ b/CirrusSearch.php @@ -550,6 +550,18 @@ // Results are cached. $wgCirrusSearchInterwikiSources = []; +/** + * Temporary special configuration for load testing the addition of interwiki + * search results to a wiki. If this value is null then nothing special + * happens, and wgCirrusSearchInterwikiSources is treated as usual. If this is + * set to a value between 0 and 1 that is treated as the % of requests to + * Special:Search that should use wgCirrusSearchInterwikiSources to make a + * query. The results of this query will not be attached to the + * SearchResultSet, and will not be displayed to the user. This is to estimate + * the effect of adding this additional load onto a search cluster. + */ +$wgCirrusSearchInterwikiLoadTest = null; + // How long to cache interwiki search results for (in seconds) $wgCirrusSearchInterwikiCacheTime = 7200; diff --git a/includes/CirrusSearch.php b/includes/CirrusSearch.php index 2ecc29e..410aa5c 100644 --- a/includes/CirrusSearch.php +++ b/includes/CirrusSearch.php @@ -403,7 +403,8 @@ $highlightingConfig ^= FullTextResultsType::HIGHLIGHT_FILE_TEXT; } - $searcher->setResultsType( new FullTextResultsType( $highlightingConfig, $config ? $config->getWikiCode() : '') ); + $resultsType = new FullTextResultsType( $highlightingConfig, $config ? $config->getWikiCode() : ''); + $searcher->setResultsType( $resultsType ); $status = $searcher->searchText( $term, $this->showSuggestion ); $this->lastSearchMetrics = $searcher->getSearchMetrics(); @@ -429,6 +430,12 @@ $iwSearch->setReturnQuery( $dumpQuery ); $iwSearch->setDumpResult( $dumpResult ); $iwSearch->setReturnExplain( $returnExplain ); + // This is not strictly true, or even really half true. It is a stand + // in for the purposes of interwiki load testing using the full + // search configuration, rather than the limited InterwikiResultsType. + // InterwikiSearcher will use this or InterwikiResultsType depending + // on it's needs. + $iwSearch->setResultsType( $resultsType ); $interwikiResults = $iwSearch->getInterwikiResults( $term ); if ( $interwikiResults !== null ) { // If we are dumping we need to convert into an array that can be appended to diff --git a/includes/InterwikiSearcher.php b/includes/InterwikiSearcher.php index b29ee43..722a62c 100644 --- a/includes/InterwikiSearcher.php +++ b/includes/InterwikiSearcher.php @@ -37,6 +37,16 @@ private $interwiki; /** + * @var bool Is the interwiki load test configured? + */ + private $isLoadTest = false; + + /** + * @var bool Is the interwiki load test enabled? + */ + private $isLoadTestEnabled = false; + + /** * Constructor * @param Connection $connection * @param int[]|null $namespaces Namespace numbers to search, or null for all of them @@ -50,7 +60,24 @@ return $namespace <= 15; } ); } - parent::__construct( $connection, 0, self::MAX_RESULTS, null, $namespaces, $user ); + + $limit = self::MAX_RESULTS; + + $loadTestPercent = $this->config->get( 'CirrusSearchInterwikiLoadTest' ); + if ($loadTestPercent !== null ) { + $this->isLoadTest = true; + + $isSpecialSearch = Title::newFromText( 'Special:Search' )->equals( + \RequestContext::getMain()->getTitle() + ); + $rand = mt_rand(0, PHP_INT_MAX)/PHP_INT_MAX; + if ( $isSpecialSearch && $rand >= $loadTestPercent ) { + $this->isLoadTestEnabled = true; + $limit = 1; + } + } + + parent::__construct( $connection, 0, $limit, null, $namespaces, $user ); } /** @@ -68,6 +95,11 @@ if ( !$sources ) { return null; } + + if ( $this->isLoadTest && !$this->isLoadTestEnabled ) { + return null; + } + $this->searchContext->setCacheTtl( $this->config->get( 'CirrusSearchInterwikiCacheTime' ) ); @@ -84,8 +116,13 @@ $searches = []; $resultsTypes = []; foreach ( $sources as $interwiki => $index ) { - $resultsTypes[$interwiki] = new InterwikiResultsType( $interwiki ); - $this->setResultsType( $resultsTypes[$interwiki] ); + // Note that this is a hack, $this->resultsType is not properly + // specialized to the interwiki use case, but because we are not + // returning load test results to the users that is acceptable. + if (!$this->isLoadTestEnabled ) { + $resultsTypes[$interwiki] = new InterwikiResultsType( $interwiki ); + $this->setResultsType( $resultsTypes[$interwiki] ); + } $this->indexBaseName = $index; $this->searchContext = clone $context; $search = $this->buildSearch(); @@ -101,7 +138,11 @@ return null; } - return array_merge( $retval, $results->getValue() ); + if ($loadTest) { + return null; + } else { + return array_merge( $retval, $results->getValue() ); + } } /** -- To view, visit https://gerrit.wikimedia.org/r/319485 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Idb297c66dd626766a2ec6cb2b75d3c5a2a256afe Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/extensions/CirrusSearch Gerrit-Branch: master Gerrit-Owner: EBernhardson <ebernhard...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits