BryanDavis has uploaded a new change for review.
https://gerrit.wikimedia.org/r/89325
Change subject: Add HTCP rate limiting to SquidUpdate
......................................................................
Add HTCP rate limiting to SquidUpdate
Add a parameter to SquidUpdate, SquidUpdate::purge() and
SquidUpdate::HTCPPurge() that allows the caller to trigger a delay
between generation of HTCP UDP packets. This can be used to effectively
throttle the rate at which UDP packets are emitted onto the network
which is useful in the case of batch operations where hundred or
thousands of purge requests are being requested. By default no rate
limiting is introduced.
Also update the purgeChangedPages.php maintenance script to call
SquidUpdate with a 5000 microsecond delay between packets. This will
cap the UDP transmission rate for the script at 200/second.
Bug: 55632
Change-Id: Ibfc54b1767f145098465404a2b23cd92852e41fd
---
M includes/cache/SquidUpdate.php
M maintenance/purgeChangedPages.php
2 files changed, 24 insertions(+), 7 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core
refs/changes/25/89325/1
diff --git a/includes/cache/SquidUpdate.php b/includes/cache/SquidUpdate.php
index d060d4b..4d1b160 100644
--- a/includes/cache/SquidUpdate.php
+++ b/includes/cache/SquidUpdate.php
@@ -26,13 +26,14 @@
* @ingroup Cache
*/
class SquidUpdate {
- var $urlArr, $mMaxTitles;
+ var $urlArr, $mMaxTitles, $htcpDelay;
/**
* @param $urlArr array
* @param $maxTitles bool|int
+ * @param $htcpDelay int Number of microseconds to delay between HTCP
packets
*/
- function __construct( $urlArr = array(), $maxTitles = false ) {
+ function __construct( $urlArr = array(), $maxTitles = false, $htcpDelay
= null ) {
global $wgMaxSquidPurgeTitles;
if ( $maxTitles === false ) {
$this->mMaxTitles = $wgMaxSquidPurgeTitles;
@@ -44,6 +45,7 @@
$urlArr = array_slice( $urlArr, 0, $this->mMaxTitles );
}
$this->urlArr = $urlArr;
+ $this->htcpDelay = $htcpDelay;
}
/**
@@ -110,7 +112,7 @@
* Purges the list of URLs passed to the constructor
*/
function doUpdate() {
- SquidUpdate::purge( $this->urlArr );
+ SquidUpdate::purge( $this->urlArr, $this->htcpDelay );
}
/**
@@ -120,9 +122,10 @@
* XXX report broken Squids per mail or log
*
* @param $urlArr array
+ * @param $htcpDelay int Number of microseconds to delay between HTCP
packets
* @return void
*/
- static function purge( $urlArr ) {
+ static function purge( $urlArr, $htcpDelay = null ) {
global $wgSquidServers, $wgHTCPRouting;
if ( !$urlArr ) {
@@ -132,7 +135,7 @@
wfDebugLog( 'squid', __METHOD__ . ': ' . implode( ' ', $urlArr
) . "\n" );
if ( $wgHTCPRouting ) {
- SquidUpdate::HTCPPurge( $urlArr );
+ SquidUpdate::HTCPPurge( $urlArr, $htcpDelay );
}
wfProfileIn( __METHOD__ );
@@ -162,10 +165,14 @@
}
/**
+ * Send Hyper Text Caching Protocol (HTCP) CLR (clear) packets via
muticast
+ * UDP.
+ *
* @throws MWException
* @param $urlArr array
+ * @param $htcpDelay int Number of microseconds to delay between HTCP
packets
*/
- static function HTCPPurge( $urlArr ) {
+ static function HTCPPurge( $urlArr, $htcpDelay = null ) {
global $wgHTCPRouting, $wgHTCPMulticastTTL;
wfProfileIn( __METHOD__ );
@@ -245,6 +252,10 @@
socket_sendto( $conn, $htcpPacket, $htcpLen, 0,
$subconf['host'], $subconf['port'] );
}
+
+ if ( $htcpDelay !== null ) {
+ usleep( $htcpDelay );
+ }
}
wfProfileOut( __METHOD__ );
}
diff --git a/maintenance/purgeChangedPages.php
b/maintenance/purgeChangedPages.php
index e1c6ab6..450a3ba 100644
--- a/maintenance/purgeChangedPages.php
+++ b/maintenance/purgeChangedPages.php
@@ -35,6 +35,12 @@
*/
class PurgeChangedPages extends Maintenance {
+ /**
+ * Number of microseconds to sleep between HTCP purge packets.
+ * @var int
+ */
+ const RATE_LIMIT = 5000;
+
public function __construct() {
parent::__construct();
$this->mDescription = 'Send purge requests for edits in date
range to squid/varnish';
@@ -135,7 +141,7 @@
}
// Send batch of purge requests out to squids
- $squid = new SquidUpdate( $urls );
+ $squid = new SquidUpdate( $urls, count( $urls ),
self::RATE_LIMIT );
$squid->doUpdate();
}
--
To view, visit https://gerrit.wikimedia.org/r/89325
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ibfc54b1767f145098465404a2b23cd92852e41fd
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/core
Gerrit-Branch: master
Gerrit-Owner: BryanDavis <bda...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
