jenkins-bot has submitted this change and it was merged.
Change subject: Per user request limits
......................................................................
Per user request limits
Needs poolcounter's Ie282b8486c7bad451fbc5fb9a8274c6e01a728a7 before we can
set $wgPoolCounterConf['CirrusSearch-PerUser'] otherwise we just abuse the
poor poolcounter. If we don't set $wgPoolCounterConf['CirrusSearch-PerUser']
then this is just a noop.
Adds some testing code that detects if the poolcounter is setup and configures
cirrus to use it if it is.
Change-Id: Iedda2d1f39bc70bb8103d0fdfb1297708c960766
---
M README
M i18n/en.json
M includes/Searcher.php
M includes/Util.php
M tests/jenkins/Jenkins.php
5 files changed, 110 insertions(+), 29 deletions(-)
Approvals:
Chad: Looks good to me, approved
jenkins-bot: Verified
diff --git a/README b/README
index 4986715..68fcbf8 100644
--- a/README
+++ b/README
@@ -108,26 +108,32 @@
elasticsearch. You can do this by installing the PoolCounter extension and
then configuring it in
LocalSettings.php like so:
require_once( "$IP/extensions/PoolCounter/PoolCounterClient.php");
- $wgPoolCounterConf = array(
- 'CirrusSearch-Search' => array( // Configuration for non-prefix
non-regex searches
- 'class' => 'PoolCounter_Client',
- 'timeout' => 30,
- 'workers' => 25,
- 'maxqueue' => 10,
- ),
- 'CirrusSearch-Prefix' => array( // Configuration for prefix searches
- 'class' => 'PoolCounter_Client',
- 'timeout' => 10,
- 'workers' => 50,
- 'maxqueue' => 10,
- ),
- 'CirrusSearch-Regex' => array( // Configuration for regex searches
- 'class' => 'PoolCounter_Client',
- 'timeout' => 30,
- 'workers' => 10,
- 'maxqueue' => 0,
- ),
- 'CirrusSearch-NamespaceLookup' => array( // Configuration for funky
namespace lookups
+ // Configuration for standard searches.
+ $wgPoolCounterConf[ 'CirrusSearch-Search' ] = array(
+ 'class' => 'PoolCounter_Client',
+ 'timeout' => 30,
+ 'workers' => 25,
+ 'maxqueue' => 50,
+ );
+ // Configuration for prefix searches. These are usually quite quick and
+ // plentiful.
+ $wgPoolCounterConf[ 'CirrusSearch-Prefix' ] = array(
+ 'class' => 'PoolCounter_Client',
+ 'timeout' => 10,
+ 'workers' => 50,
+ 'maxqueue' => 100,
+ );
+ // Configuration for regex searches. These are slow and use lots of resources
+ // so we only allow a few at a time.
+ $wgPoolCounterConf[ 'CirrusSearch-Regex' ] = array(
+ 'class' => 'PoolCounter_Client',
+ 'timeout' => 30,
+ 'workers' => 10,
+ 'maxqueue' => 10,
+ );
+ // Configuration for funky namespace lookups. These should be reasonably fast
+ // and reasonably rare.
+ $wgPoolCounterConf[ 'CirrusSearch-NamespaceLookup' ] = array(
'class' => 'PoolCounter_Client',
'timeout' => 10,
'workers' => 20,
@@ -135,6 +141,15 @@
),
);
+If you have a version of the pool counter with
Ie282b8486c7bad451fbc5fb9a8274c6e01a728a7
+you can also enable per user request limits:
+ // Each user (or remote IP) can only do 5 concurrent queries.
+ $wgPoolCounterConf[ 'CirrusSearch-PerUser' ] = array(
+ 'class' => 'PoolCounter_Client',
+ 'timeout' => 0,
+ 'workers' => 5,
+ 'maxqueue' => 5,
+ );
Upgrading
---------
@@ -257,6 +272,7 @@
This can take some time but it produces a clean development environment in a
virtual machine
that has everything required to run Cirrus.
+
Hooks
-----
CirrusSearch provides hooks that other extensions can make use of to extend
the core schema and
@@ -284,5 +300,6 @@
Licensing information
---------------------
-CirrusSearch makes use of the Elastica library to connect to elasticsearch
<http://elastica.io/>.
-It is Apache licensed and you can read the license Elastica/LICENSE.txt.
+CirrusSearch makes use of the Elastica extension containing the Elastica
library to connect
+to Elasticsearch <http://elastica.io/>. It is Apache licensed and you can read
the license
+Elastica/LICENSE.txt.
diff --git a/i18n/en.json b/i18n/en.json
index 87de93a..3ed8ecb 100644
--- a/i18n/en.json
+++ b/i18n/en.json
@@ -6,6 +6,8 @@
"cirrussearch-backend-error": "We could not complete your search due to
a temporary problem. Please try again later.",
"cirrussearch-parse-error": "Query was not understood. Please make it
simpler. The query was logged to improve the search system.",
"cirrussearch-too-busy-error": "Search is currently too busy. Please
try again later.",
+ "cirrussearch-too-busy-for-you-anonymous-error": "You have too many
concurrent searches running. If you are sharing an IP address with other users
you can log in to get your own limits.",
+ "cirrussearch-too-busy-for-you-logged-in-error": "You have too many
concurrent searches running.",
"cirrussearch-regex-syntax-error": "Regular expression syntax error at
$2: $1",
"cirrussearch-regex-too-busy-error": "Too many regular expression
searches currently running. Please try again later.",
"cirrussearch-regex-too-complex-error": "Regular expression is too
complex. Learn more about simplifying it
[[mw:Special:MyLanguage/Help:CirrusSearch/RegexTooComplex|here]].",
diff --git a/includes/Searcher.php b/includes/Searcher.php
index f9c0b0d..35c20d0 100644
--- a/includes/Searcher.php
+++ b/includes/Searcher.php
@@ -830,6 +830,7 @@
$indexBaseName = $this->indexBaseName;
return Util::doPoolCounterWork(
'CirrusSearch-Search',
+ $this->user,
function() use ( $searcher, $pageIds, $sourceFiltering,
$indexType, $indexBaseName ) {
try {
global
$wgCirrusSearchClientSideSearchTimeout;
@@ -857,6 +858,7 @@
$indexBaseName = $this->indexBaseName;
return Util::doPoolCounterWork(
'CirrusSearch-NamespaceLookup',
+ $this->user,
function() use ( $searcher, $name, $indexBaseName ) {
try {
$searcher->start( "lookup namespace for
$name" );
@@ -1083,8 +1085,10 @@
// Perform the search
$searcher = $this;
wfProfileIn( __METHOD__ . '-execute' );
+ $user = $this->user;
$result = Util::doPoolCounterWork(
$poolCounterType,
+ $this->user,
function() use ( $searcher, $search, $description ) {
try {
$searcher->start( $description );
@@ -1093,9 +1097,13 @@
return $searcher->failure( $e );
}
},
- function( $error, $key ) use ( $type, $description ) {
+ function( $error, $key ) use ( $type, $description,
$user ) {
wfLogWarning( "Pool error on key $key during
$description: $error" );
if ( $error === 'pool-queuefull' ) {
+ if ( strpos( $key,
'CirrusSearch:_per_user' ) === 0 ) {
+ $loggedIn = $user->isLoggedIn()
? 'logged-in' : 'anonymous';
+ return Status::newFatal(
"cirrussearch-too-busy-for-you-{$loggedIn}-error" );
+ }
if ( $type === 'regex' ) {
return Status::newFatal(
'cirrussearch-regex-too-busy-error' );
}
diff --git a/includes/Util.php b/includes/Util.php
index 0b7bf47..d4f9c46 100644
--- a/includes/Util.php
+++ b/includes/Util.php
@@ -90,31 +90,45 @@
* Wraps the complex pool counter interface to force the single call
pattern
* that Cirrus always uses.
* @param $type same as type parameter on PoolCounter::factory
+ * @param $user the user
* @param $workCallback callback when pool counter is aquired. Called
with
* no parameters.
* @param $errorCallback optional callback called on errors. Called
with
* the error string and the key as parameters. If left undefined
defaults
* to a function that returns a fatal status and logs an warning.
*/
- public static function doPoolCounterWork( $type, $workCallback,
$errorCallback = null ) {
+ public static function doPoolCounterWork( $type, $user, $workCallback,
$errorCallback = null ) {
global $wgCirrusSearchPoolCounterKey;
// By default the pool counter allows you to lock the same key
with
// multiple types. That might be useful but it isn't how
Cirrus thinks.
// Instead, all keys are scoped to their type.
- $key = "$type:$wgCirrusSearchPoolCounterKey";
+ $perUserKey = md5( $user->getName() );
+ $perUserKey = "nowait:CirrusSearch:_per_user:$perUserKey";
+ $globalKey = "$type:$wgCirrusSearchPoolCounterKey";
if ( $errorCallback === null ) {
$errorCallback = function( $error, $key ) {
wfLogWarning( "Pool error on $key: $error" );
return Status::newFatal(
'cirrussearch-backend-error' );
};
}
- $work = new PoolCounterWorkViaCallback( $type, $key, array(
- 'doWork' => $workCallback,
- 'error' => function( $status ) use ( $errorCallback,
$key ) {
+ $errorHandler = function( $key ) use ( $errorCallback ) {
+ return function( $status ) use ( $errorCallback, $key )
{
$status = $status->getErrorsArray();
return $errorCallback( $status[ 0 ][ 0 ], $key
);
- }
+ };
+ };
+ $work = new PoolCounterWorkViaCallback( 'CirrusSearch-PerUser',
$perUserKey, array(
+ 'doWork' => function() use ( $type, $globalKey,
$workCallback, $errorHandler ) {
+ // Now that we have the per user lock lets get
the operation lock.
+ // Note that this could block, causing the user
to wait in line with their lock held.
+ $work = new PoolCounterWorkViaCallback( $type,
$globalKey, array(
+ 'doWork' => $workCallback,
+ 'error' => $errorHandler( $globalKey ),
+ ) );
+ return $work->execute();
+ },
+ 'error' => $errorHandler( $perUserKey ),
) );
return $work->execute();
}
diff --git a/tests/jenkins/Jenkins.php b/tests/jenkins/Jenkins.php
index 9c94ba2..6091fcf 100644
--- a/tests/jenkins/Jenkins.php
+++ b/tests/jenkins/Jenkins.php
@@ -87,6 +87,46 @@
$wgCirrusSearchLanguageWeight[ 'user' ] = 10.0;
$wgCirrusSearchLanguageWeight[ 'wiki' ] = 5.0;
+if ( class_exists( 'PoolCounter_Client' ) ) {
+ // If the pool counter is around set up prod like pool counter settings
+ $wgPoolCounterConf[ 'CirrusSearch-Search' ] = array(
+ 'class' => 'PoolCounter_Client',
+ 'timeout' => 15,
+ 'workers' => 432,
+ 'maxqueue' => 600,
+ );
+ // Super common and mostly fast
+ $wgPoolCounterConf[ 'CirrusSearch-Prefix' ] = array(
+ 'class' => 'PoolCounter_Client',
+ 'timeout' => 15,
+ 'workers' => 432,
+ 'maxqueue' => 600,
+ );
+ // Regex searches are much heavier then regular searches so we limit the
+ // concurrent number.
+ $wgPoolCounterConf[ 'CirrusSearch-Regex' ] = array(
+ 'class' => 'PoolCounter_Client',
+ 'timeout' => 60,
+ 'workers' => 10,
+ 'maxqueue' => 20,
+ );
+ // These should be very very fast and reasonably rare
+ $wgPoolCounterConf[ 'CirrusSearch-NamespaceLookup' ] = array(
+ 'class' => 'PoolCounter_Client',
+ 'timeout' => 5,
+ 'workers' => 50,
+ 'maxqueue' => 200,
+ );
+ // Can't be enabled until poolcounter gets
Ie282b8486c7bad451fbc5fb9a8274c6e01a728a7.
+ // TODO enable this.
+ // $wgPoolCounterConf[ 'CirrusSearch-PerUser' ] = array(
+ // 'class' => 'PoolCounter_Client',
+ // 'timeout' => 0,
+ // 'workers' => 1,
+ // 'maxqueue' => 1,
+ // );
+}
+
class Jenkins {
/**
* Installs maintenance scripts that provide a clean Elasticsearch
index for testing.
--
To view, visit https://gerrit.wikimedia.org/r/177552
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Iedda2d1f39bc70bb8103d0fdfb1297708c960766
Gerrit-PatchSet: 6
Gerrit-Project: mediawiki/extensions/CirrusSearch
Gerrit-Branch: master
Gerrit-Owner: Manybubbles <[email protected]>
Gerrit-Reviewer: Chad <[email protected]>
Gerrit-Reviewer: Manybubbles <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
