Ottomata has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/380804 )
Change subject: [WIP] Set up separate druid public-eqiad cluster.
......................................................................
[WIP] Set up separate druid public-eqiad cluster.
This should not be merged yet, we need to do a lot of manual work
to prepare these servers.
NOTE: It is safe to rename the 'druid-eqiad' Zookeeper cluster
to 'druid-analytics-eqiad', because it is only used in puppet.
There is nothing in zookeeper or druid configurations that
refer to the Zookeeper cluster name.
This also sets up a new Zookeeper cluster colocated on the
druid public-eqiad nodes called 'druid-public-eqiad'.
Bug: T176223
Change-Id: I8624fbc402105cc44818a2ee8d0db7dbab3526ee
---
M hieradata/common.yaml
M hieradata/role/common/druid/analytics/worker.yaml
M manifests/site.pp
M modules/network/manifests/constants.pp
4 files changed, 31 insertions(+), 9 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/04/380804/1
diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index 1822f98..040d968 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -439,14 +439,22 @@
conf2002.codfw.wmnet: '2002'
conf2003.codfw.wmnet: '2003'
- # ZK cluster for Druid (in Analytics cluster),
- # colocated with Druid.
- druid-eqiad:
+ # ZK cluster for Druid analytics-eqiad cluster (non public),
+ # colocated on druid hosts.
+ druid-analytics-eqiad:
hosts:
druid1001.eqiad.wmnet: '1001'
druid1002.eqiad.wmnet: '1002'
druid1003.eqiad.wmnet: '1003'
+ # ZK cluster for Druid public-eqiad cluster, (for AQS, wikistats, etc.)
+ # colocated on druid hosts.
+ druid-public-eqiad:
+ hosts:
+ druid1004.eqiad.wmnet: '1004'
+ druid1005.eqiad.wmnet: '1005'
+ druid1006.eqiad.wmnet: '1006'
+
# Used to sync the setting between all Kafka clusters and clients.
kafka_message_max_bytes: 4194304
diff --git a/hieradata/role/common/druid/analytics/worker.yaml
b/hieradata/role/common/druid/analytics/worker.yaml
index e5715a8..7e4dc04 100644
--- a/hieradata/role/common/druid/analytics/worker.yaml
+++ b/hieradata/role/common/druid/analytics/worker.yaml
@@ -6,7 +6,7 @@
# Druid nodes get their own Zookeeper cluster to isolate them
# from the production ones.
# Configure the zookeeper profile.
-profile::zookeeper::cluster_name: druid-eqiad
+profile::zookeeper::cluster_name: druid-analytics-eqiad
# Don't page if a zookeeper server in this cluster goes down.
profile::zookeeper::is_critical: false
# Max number of connections per IP for Zookeeper
@@ -33,7 +33,7 @@
# The logical name of this druid cluster
profile::druid::common::druid_cluster_name: analytics-eqiad
# The logical name of the zookeeper cluster that druid should use
-profile::druid::common::zookeeper_cluster_name: druid-eqiad
+profile::druid::common::zookeeper_cluster_name: druid-analytics-eqiad
# Make druid build an extension composed of CDH jars.
profile::druid::common::use_cdh: true
diff --git a/manifests/site.pp b/manifests/site.pp
index d5ebe1c..ca8a884 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -771,9 +771,21 @@
interface::add_ip6_mapped { 'main': }
}
-# Analytics Druid servers.
+# Druid analytics-eqiad (non public) servers.
+# These power internal backends and queries.
# https://wikitech.wikimedia.org/wiki/Analytics/Data_Lake#Druid
-node /^druid100[123456].eqiad.wmnet$/ {
+node /^druid100[123].eqiad.wmnet$/ {
+ role(druid::analytics::worker)
+
+ include ::base::firewall
+ include ::standard
+}
+
+
+# Druid public-eqiad servers.
+# These power AQS and wikistats 2.0 and contain non sensitive datasets.
+# https://wikitech.wikimedia.org/wiki/Analytics/Data_Lake#Druid
+node /^druid100[456].eqiad.wmnet$/ {
role(druid::analytics::worker)
include ::base::firewall
diff --git a/modules/network/manifests/constants.pp
b/modules/network/manifests/constants.pp
index 45c6fac..0d726ea 100644
--- a/modules/network/manifests/constants.pp
+++ b/modules/network/manifests/constants.pp
@@ -160,20 +160,22 @@
'10.64.53.21', #
analytics1002.eqiad.wmnet
'2620:0:861:108:f21f:afff:fee8:bc3f', #
analytics1002.eqiad.wmnet
],
- 'druid_hosts' => [
+ 'druid_analytics_hosts' => [
'10.64.5.101', #
druid1001.eqiad.wmnet
'2620:0:861:104:1e98:ecff:fe29:e298', #
druid1001.eqiad.wmnet
'10.64.36.102', #
druid1002.eqiad.wmnet
'2620:0:861:106:1602:ecff:fe06:8bec', #
druid1002.eqiad.wmnet
'10.64.53.103', #
druid1003.eqiad.wmnet
'2620:0:861:108:1e98:ecff:fe29:e278', #
druid1003.eqiad.wmnet
+ ],
+ 'druid_public_hosts' => [
'10.64.5.24', #
druid1004.eqiad.wmnet
'2620:0:861:104:1a66:daff:feac:87a1', #
druid1004.eqiad.wmnet
'10.64.21.109', #
druid1005.eqiad.wmnet
'2620:0:861:105:1a66:daff:feae:36fb', #
druid1005.eqiad.wmnet
'10.64.53.32', #
druid1006.eqiad.wmnet
'2620:0:861:108:1a66:daff:feac:75cd', #
druid1006.eqiad.wmnet
- ],
+ ],
'cache_misc' => [
'10.64.32.97', # cp1045.eqiad.wmnet
'2620:0:861:103:10:64:32:97', # cp1045.eqiad.wmnet
--
To view, visit https://gerrit.wikimedia.org/r/380804
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I8624fbc402105cc44818a2ee8d0db7dbab3526ee
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <ao...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
