jenkins-bot has submitted this change and it was merged.
Change subject: Use ProductionServices for the jobqueue configuration
......................................................................
Use ProductionServices for the jobqueue configuration
Also update the codfw configs to match reality.
Bug: T114273
Change-Id: I141366702db66fbb82a70970e4fc4d5a911f218b
---
M docroot/noc/createTxtFileSymlinks.sh
M wmf-config/CommonSettings.php
M wmf-config/ProductionServices.php
D wmf-config/jobqueue-codfw.php
D wmf-config/jobqueue-eqiad.php
A wmf-config/jobqueue.php
6 files changed, 101 insertions(+), 180 deletions(-)
Approvals:
Aaron Schulz: Looks good to me, but someone else must approve
Thcipriani: Looks good to me, approved
jenkins-bot: Verified
diff --git a/docroot/noc/createTxtFileSymlinks.sh
b/docroot/noc/createTxtFileSymlinks.sh
index f572d39..1a915af 100755
--- a/docroot/noc/createTxtFileSymlinks.sh
+++ b/docroot/noc/createTxtFileSymlinks.sh
@@ -25,7 +25,7 @@
HHVMRequestInit.php
import.php
interwiki.php
- jobqueue-eqiad.php
+ jobqueue.php
jobqueue-labs.php
LegalContactPages.php
AffComContactPages.php
diff --git a/wmf-config/CommonSettings.php b/wmf-config/CommonSettings.php
index f6fc033..65ff9f3 100644
--- a/wmf-config/CommonSettings.php
+++ b/wmf-config/CommonSettings.php
@@ -952,7 +952,7 @@
# Cluster-dependent files for job queue and job queue aggregator
require $wmfRealm === 'labs'
? "$wmfConfigDir/jobqueue-labs.php"
- : "$wmfConfigDir/jobqueue-{$wmfDatacenter}.php";
+ : "$wmfConfigDir/jobqueue.php";
}
if ( $wgDBname == 'nostalgiawiki' ) {
diff --git a/wmf-config/ProductionServices.php
b/wmf-config/ProductionServices.php
index c5e2766..b83a158 100644
--- a/wmf-config/ProductionServices.php
+++ b/wmf-config/ProductionServices.php
@@ -86,6 +86,57 @@
'rdb3' => '10.192.32.20',
);
+### Jobqueue redis
+// Note: for each host:port combination, the local fallback slave is
+// the next server, so rdb1001:6379 => rdb1002:6379 and so on.
+// Note: on server failure, partition masters should be switched to the slave
+// Note: MediaWiki will fail-over to other shards when one is down. On master
+// failure, it is best to either do nothing or just disable the whole shard
+// until the master is fixed or full fail-over is done. Proper fail over
+// requires changing the slave to stop slaving the old master before updating
+// the MediaWiki config to direct traffic there.
+// Do NOT remove entries from here if they are still present in
'partitionsBySection'
+// in wmf-config/jobqueue.php
+$wmfAllServices['eqiad']['jobqueue_redis'] = array(
+ 'rdb1-6379' => 'rdb1001.eqiad.wmnet:6379',
+ 'rdb1-6380' => 'rdb1001.eqiad.wmnet:6380',
+ 'rdb1-6381' => 'rdb1001.eqiad.wmnet:6381',
+ 'rdb2-6379' => 'rdb1003.eqiad.wmnet:6379',
+ 'rdb2-6380' => 'rdb1003.eqiad.wmnet:6380',
+ 'rdb2-6381' => 'rdb1003.eqiad.wmnet:6381',
+ 'rdb3-6379' => 'rdb1007.eqiad.wmnet:6379',
+ 'rdb3-6380' => 'rdb1007.eqiad.wmnet:6380',
+ 'rdb3-6381' => 'rdb1007.eqiad.wmnet:6381',
+ 'rdb4-6379' => 'rdb1005.eqiad.wmnet:6379',
+ 'rdb4-6380' => 'rdb1005.eqiad.wmnet:6380',
+ 'rdb4-6381' => 'rdb1005.eqiad.wmnet:6381',
+);
+$wmfAllServices['codfw']['jobqueue_redis'] = array(
+ 'rdb1-6379' => 'rdb2001.eqiad.wmnet:6379',
+ 'rdb1-6380' => 'rdb2001.eqiad.wmnet:6380',
+ 'rdb1-6381' => 'rdb2001.eqiad.wmnet:6381',
+ 'rdb2-6379' => 'rdb2003.eqiad.wmnet:6379',
+ 'rdb2-6380' => 'rdb2003.eqiad.wmnet:6380',
+ 'rdb2-6381' => 'rdb2003.eqiad.wmnet:6381',
+ 'rdb3-6379' => 'rdb2005.eqiad.wmnet:6479',
+ 'rdb3-6380' => 'rdb2005.eqiad.wmnet:6480',
+ 'rdb3-6381' => 'rdb2005.eqiad.wmnet:6481',
+ 'rdb4-6379' => 'rdb2005.eqiad.wmnet:6379',
+ 'rdb4-6380' => 'rdb2005.eqiad.wmnet:6380',
+ 'rdb4-6381' => 'rdb2005.eqiad.wmnet:6381',
+);
+$wmfAllServices['eqiad']['jobqueue_aggregator'] = array(
+ 'rdb1001.eqiad.wmnet:6378', // preferred
+ 'rdb1003.eqiad.wmnet:6378', // fallback
+ 'rdb1005.eqiad.wmnet:6378', // fallback
+ 'rdb1007.eqiad.wmnet:6378', // fallback
+);
+$wmfAllServices['codfw']['jobqueue_aggregator'] = array(
+ 'rdb2001.eqiad.wmnet:6378', // preferred
+ 'rdb2003.eqiad.wmnet:6378', // fallback
+ 'rdb2005.eqiad.wmnet:6378', // fallback
+);
+
# Shorthand when we have no master-slave situation to keep into account
$wmfLocalServices = $wmfAllServices[$wmfDatacenter];
diff --git a/wmf-config/jobqueue-codfw.php b/wmf-config/jobqueue-codfw.php
deleted file mode 100644
index d79ef9e..0000000
--- a/wmf-config/jobqueue-codfw.php
+++ /dev/null
@@ -1,56 +0,0 @@
-<?php
-# WARNING: This file is publicly viewable on the web. Do not put private data
here.
-
-// Note: on server failure, partition masters should be switched to the slave
-// Note: MediaWiki will fail-over to other shards when one is down. On master
-// failure, it is best to either do nothing or just disable the whole shard
-// until the master is fixed or full fail-over is done. Proper fail over
-// requires changing the slave to stop slaving the old master before updating
-// the MediaWiki config to direct traffic there.
-$wgJobTypeConf['default'] = array(
- 'class' => 'JobQueueFederated',
- // Do NOT remove entries from here if they are still present in
'partitionsBySection'
- 'configByPartition' => array(
- 'rdb1' => array(
- 'class' => 'JobQueueRedis',
- 'redisServer' => '10.192.0.119', # rdb2001 (master)
- #'redisServer' => '10.192.16.122', # rdb2003 (slave)
- 'redisConfig' => array(
- 'connectTimeout' => 2,
- 'password' => $wmgRedisPassword,
- 'compression' => 'gzip'
- ),
- 'daemonized' => true
- ),
- 'rdb2' => array(
- 'class' => 'JobQueueRedis',
- 'redisServer' => '10.192.0.120', # rdb2002 (master)
- #'redisServer' => '10.192.16.123', # rdb2004 (slave)
- 'redisConfig' => array(
- 'connectTimeout' => 2,
- 'password' => $wmgRedisPassword,
- 'compression' => 'gzip'
- ),
- 'daemonized' => true
- ),
- ),
- 'sectionsByWiki' => array(), // default
- // Weights for partitions in use: use this to depool redis masters
- 'partitionsBySection' => array(
- 'default' => array( 'rdb1' => 50, 'rdb2' => 50 ),
- )
-);
-// Note: on server failure, this should be changed to any other redis server
-$wgJobQueueAggregator = array(
- 'class' => 'JobQueueAggregatorRedis',
- 'redisServers' => array( // all after the first are fallbacks
- '10.192.0.119', # rdb2001
- '10.192.0.120', # rdb2002
- ),
- 'redisConfig' => array(
- 'connectTimeout' => 2,
- 'password' => $wmgRedisPassword,
- )
-);
-
-$wgJobSerialCommitThreshold = .100; // 100 ms
diff --git a/wmf-config/jobqueue-eqiad.php b/wmf-config/jobqueue-eqiad.php
deleted file mode 100644
index d14c675..0000000
--- a/wmf-config/jobqueue-eqiad.php
+++ /dev/null
@@ -1,122 +0,0 @@
-<?php
-# WARNING: This file is publicly viewable on the web. Do not put private data
here.
-
-/** @var string $wmgRedisPassword From PrivateSettings.php */
-
-$wmgRedisQueueBaseConfig = array(
- 'class' => 'JobQueueRedis',
- 'redisConfig' => array(
- 'connectTimeout' => .300,
- 'password' => $wmgRedisPassword,
- 'compression' => 'gzip',
- #'persistent' => defined( 'MEDIAWIKI_JOB_RUNNER' )
- ),
- 'daemonized' => true
-);
-
-// Note: on server failure, partition masters should be switched to the slave
-// Note: MediaWiki will fail-over to other shards when one is down. On master
-// failure, it is best to either do nothing or just disable the whole shard
-// until the master is fixed or full fail-over is done. Proper fail over
-// requires changing the slave to stop slaving the old master before updating
-// the MediaWiki config to direct traffic there.
-$wgJobTypeConf['default'] = array(
- 'class' => 'JobQueueFederated',
- // Do NOT remove entries from here if they are still present in
'partitionsBySection'
- 'configByPartition' => array(
- # rdb 1
-
- 'rdb1-6379' => array(
- 'redisServer' => 'rdb1001.eqiad.wmnet:6379', #
master
- #'redisServer' => 'rdb1002.eqiad.wmnet:6379', #
slave
- ) + $wmgRedisQueueBaseConfig,
- 'rdb1-6380' => array(
- 'redisServer' => 'rdb1001.eqiad.wmnet:6380', #
master
- #'redisServer' => 'rdb1002.eqiad.wmnet:6380', #
slave
- ) + $wmgRedisQueueBaseConfig,
- 'rdb1-6381' => array(
- 'redisServer' => 'rdb1001.eqiad.wmnet:6381', #
master
- #'redisServer' => 'rdb1002.eqiad.wmnet:6381', #
slave
- ) + $wmgRedisQueueBaseConfig,
-
- # rdb2
-
- 'rdb2-6379' => array(
- 'redisServer' => 'rdb1003.eqiad.wmnet:6379', #
master
- #'redisServer' => 'rdb1004.eqiad.wmnet:6379', #
slave
- ) + $wmgRedisQueueBaseConfig,
- 'rdb2-6380' => array(
- 'redisServer' => 'rdb1003.eqiad.wmnet:6380', #
master
- #'redisServer' => 'rdb1004.eqiad.wmnet:6380', #
slave
- ) + $wmgRedisQueueBaseConfig,
- 'rdb2-6381' => array(
- 'redisServer' => 'rdb1003.eqiad.wmnet:6381', #
master
- #'redisServer' => 'rdb1004.eqiad.wmnet:6381', #
slave
- ) + $wmgRedisQueueBaseConfig,
-
- # rdb3
-
- 'rdb3-6379' => array(
- 'redisServer' => 'rdb1007.eqiad.wmnet:6379', #
master
- #'redisServer' => 'rdb1008.eqiad.wmnet:6379', #
slave
- ) + $wmgRedisQueueBaseConfig,
- 'rdb3-6380' => array(
- 'redisServer' => 'rdb1007.eqiad.wmnet:6380', #
master
- #'redisServer' => 'rdb1008.eqiad.wmnet:6380', #
slave
- ) + $wmgRedisQueueBaseConfig,
- 'rdb3-6381' => array(
- 'redisServer' => 'rdb1007.eqiad.wmnet:6381', #
master
- #'redisServer' => 'rdb1008.eqiad.wmnet:6381', #
slave
- ) + $wmgRedisQueueBaseConfig,
-
- # rdb4
-
- 'rdb4-6379' => array(
- 'redisServer' => 'rdb1005.eqiad.wmnet:6379', #
master
- #'redisServer' => 'rdb1006.eqiad.wmnet:6379', #
slave
- ) + $wmgRedisQueueBaseConfig,
- 'rdb4-6380' => array(
- 'redisServer' => 'rdb1005.eqiad.wmnet:6380', #
master
- #'redisServer' => 'rdb1006.eqiad.wmnet:6380', #
slave
- ) + $wmgRedisQueueBaseConfig,
- 'rdb4-6381' => array(
- 'redisServer' => 'rdb1005.eqiad.wmnet:6381', #
master
- #'redisServer' => 'rdb1006.eqiad.wmnet:6381', #
slave
- ) + $wmgRedisQueueBaseConfig,
- ),
- 'sectionsByWiki' => array(), // default
- // Weights for partitions in use: use this to depool redis masters
- 'partitionsBySection' => array(
- 'default' => array(
- 'rdb1-6379' => 50,
- 'rdb1-6380' => 50,
- 'rdb1-6381' => 50,
- 'rdb2-6379' => 50,
- 'rdb2-6380' => 50,
- 'rdb2-6381' => 50,
- 'rdb3-6379' => 50,
- 'rdb3-6380' => 50,
- 'rdb3-6381' => 50,
- 'rdb4-6379' => 50,
- 'rdb4-6380' => 50,
- 'rdb4-6381' => 50,
- ),
- ),
- 'maxPartitionsTry' => 5 // always covers 2+ servers
-);
-// Note: on server failure, this should be changed to any other redis server
-$wgJobQueueAggregator = array(
- 'class' => 'JobQueueAggregatorRedis',
- 'redisServers' => array(
- 'rdb1001.eqiad.wmnet:6378', // preferred
- 'rdb1003.eqiad.wmnet:6378', // fallback
- 'rdb1005.eqiad.wmnet:6378', // fallback
- 'rdb1007.eqiad.wmnet:6378', // fallback
- ),
- 'redisConfig' => array(
- 'connectTimeout' => 0.5,
- 'password' => $wmgRedisPassword,
- )
-);
-
-$wgJobSerialCommitThreshold = .100; // 100 ms
diff --git a/wmf-config/jobqueue.php b/wmf-config/jobqueue.php
new file mode 100644
index 0000000..70192c0
--- /dev/null
+++ b/wmf-config/jobqueue.php
@@ -0,0 +1,48 @@
+<?php
+# WARNING: This file is publicly viewable on the web. Do not put private data
here.
+
+/** @var string $wmgRedisPassword From PrivateSettings.php */
+
+$wmgRedisQueueBaseConfig = array(
+ 'class' => 'JobQueueRedis',
+ 'redisConfig' => array(
+ 'connectTimeout' => .300,
+ 'password' => $wmgRedisPassword,
+ 'compression' => 'gzip',
+ #'persistent' => defined( 'MEDIAWIKI_JOB_RUNNER' )
+ ),
+ 'daemonized' => true
+);
+
+function wmfRedisConfigByPartition( $base, $partitions ) {
+ $result = array();
+ foreach ( $partitions as $partition => $addr ) {
+ $result[$partition] = array('redisServer' => $addr) + $base;
+ }
+ return $result;
+}
+
+$wgJobTypeConf['default'] = array(
+ 'class' => 'JobQueueFederated',
+ 'configByPartition' => wmfRedisConfigByPartition(
+ $wmgRedisQueueBaseConfig,
+ $wmfLocalServices['jobqueue_redis']
+ ),
+ 'sectionsByWiki' => array(), // default
+ // Weights for partitions in use: use this to depool redis masters
+ 'partitionsBySection' => array(
+ 'default' => array_fill_keys( array_keys(
$wmfLocalServices['jobqueue_redis'] ), 50 )
+ ),
+ 'maxPartitionsTry' => 5 // always covers 2+ servers
+);
+// Note: on server failure, this should be changed to any other redis server
+$wgJobQueueAggregator = array(
+ 'class' => 'JobQueueAggregatorRedis',
+ 'redisServers' => $wmfLocalServices['jobqueue_aggregator'],
+ 'redisConfig' => array(
+ 'connectTimeout' => 0.5,
+ 'password' => $wmgRedisPassword,
+ )
+);
+
+$wgJobSerialCommitThreshold = .100; // 100 ms
--
To view, visit https://gerrit.wikimedia.org/r/279350
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I141366702db66fbb82a70970e4fc4d5a911f218b
Gerrit-PatchSet: 2
Gerrit-Project: operations/mediawiki-config
Gerrit-Branch: master
Gerrit-Owner: Giuseppe Lavagetto <[email protected]>
Gerrit-Reviewer: Aaron Schulz <[email protected]>
Gerrit-Reviewer: Florianschmidtwelzow <[email protected]>
Gerrit-Reviewer: Giuseppe Lavagetto <[email protected]>
Gerrit-Reviewer: Thcipriani <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
