jenkins-bot has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/392697 )
Change subject: Reduce replication log spam from LoadBalancer
......................................................................
Reduce replication log spam from LoadBalancer
Set the logging level there to INFO. Move the ERROR level
logging to LoadMonitor, where it will trigger only on cache
regenerations, rather than every DB connection attempt.
Change-Id: I96fd513a01601544ea30a562746c49c88f84d96d
---
M includes/libs/rdbms/loadbalancer/LoadBalancer.php
M includes/libs/rdbms/loadmonitor/LoadMonitor.php
2 files changed, 30 insertions(+), 4 deletions(-)
Approvals:
Krinkle: Looks good to me, approved
jenkins-bot: Verified
diff --git a/includes/libs/rdbms/loadbalancer/LoadBalancer.php
b/includes/libs/rdbms/loadbalancer/LoadBalancer.php
index b622ddc..a67e6e9 100644
--- a/includes/libs/rdbms/loadbalancer/LoadBalancer.php
+++ b/includes/libs/rdbms/loadbalancer/LoadBalancer.php
@@ -189,6 +189,7 @@
} else {
$this->loadMonitorConfig = [ 'class' =>
'LoadMonitorNull' ];
}
+ $this->loadMonitorConfig += [ 'lagWarnThreshold' =>
$this->maxLag ];
foreach ( $params['servers'] as $i => $server ) {
$this->mLoads[$i] = $server['load'];
@@ -291,7 +292,7 @@
"Server {host} is not
replicating?", [ 'host' => $host ] );
unset( $loads[$i] );
} elseif ( $lag > $maxServerLag ) {
- $this->replLogger->warning(
+ $this->replLogger->info(
"Server {host} has {lag}
seconds of lag (>= {maxlag})",
[ 'host' => $host, 'lag' =>
$lag, 'maxlag' => $maxServerLag ]
);
diff --git a/includes/libs/rdbms/loadmonitor/LoadMonitor.php
b/includes/libs/rdbms/loadmonitor/LoadMonitor.php
index 8292c03..74c7765 100644
--- a/includes/libs/rdbms/loadmonitor/LoadMonitor.php
+++ b/includes/libs/rdbms/loadmonitor/LoadMonitor.php
@@ -45,9 +45,22 @@
/** @var float Moving average ratio (e.g. 0.1 for 10% weight to new
weight) */
private $movingAveRatio;
+ /** @var int Amount of replication lag in seconds before warnings are
logged */
+ private $lagWarnThreshold;
- const VERSION = 1; // cache key version
+ /** @var int cache key version */
+ const VERSION = 1;
+ /** @var int Default 'max lag' in seconds when unspecified */
+ const LAG_WARN_THRESHOLD = 10;
+ /**
+ * @param ILoadBalancer $lb
+ * @param BagOStuff $srvCache
+ * @param WANObjectCache $wCache
+ * @param array $options
+ * - movingAveRatio: moving average constant for server weight
updates based on lag
+ * - lagWarnThreshold: how many seconds of lag trigger warnings
+ */
public function __construct(
ILoadBalancer $lb, BagOStuff $srvCache, WANObjectCache $wCache,
array $options = []
) {
@@ -59,6 +72,9 @@
$this->movingAveRatio = isset( $options['movingAveRatio'] )
? $options['movingAveRatio']
: 0.1;
+ $this->lagWarnThreshold = isset( $options['lagWarnThreshold'] )
+ ? $options['lagWarnThreshold']
+ : self::LAG_WARN_THRESHOLD;
}
public function setLogger( LoggerInterface $logger ) {
@@ -159,9 +175,10 @@
// Scale from 10% to 100% of nominal weight
$weightScales[$i] = max( $newWeight, 0.10 );
+ $host = $this->parent->getServerName( $i );
+
if ( !$conn ) {
$lagTimes[$i] = false;
- $host = $this->parent->getServerName( $i );
$this->replLogger->error(
__METHOD__ . ": host {db_server} is
unreachable",
[ 'db_server' => $host ]
@@ -174,11 +191,19 @@
} else {
$lagTimes[$i] = $conn->getLag();
if ( $lagTimes[$i] === false ) {
- $host = $this->parent->getServerName(
$i );
$this->replLogger->error(
__METHOD__ . ": host
{db_server} is not replicating?",
[ 'db_server' => $host ]
);
+ } elseif ( $lagTimes[$i] >
$this->lagWarnThreshold ) {
+ $this->replLogger->error(
+ "Server {host} has {lag}
seconds of lag (>= {maxlag})",
+ [
+ 'host' => $host,
+ 'lag' => $lagTimes[$i],
+ 'maxlag' =>
$this->lagWarnThreshold
+ ]
+ );
}
}
--
To view, visit https://gerrit.wikimedia.org/r/392697
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I96fd513a01601544ea30a562746c49c88f84d96d
Gerrit-PatchSet: 4
Gerrit-Project: mediawiki/core
Gerrit-Branch: master
Gerrit-Owner: Aaron Schulz <asch...@wikimedia.org>
Gerrit-Reviewer: Krinkle <krinklem...@gmail.com>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
