Ottomata has uploaded a new change for review.
https://gerrit.wikimedia.org/r/209745
Change subject: Puppetize standalone cluster on test Ciscos
......................................................................
Puppetize standalone cluster on test Ciscos
This will be used for testing out Spark Streaming
Change-Id: Icea3cf8df94fb3164722d5bd05f5b3f89537189a
---
A hieradata/role/common/analytics/spark/standalone.yaml
M manifests/role/analytics/spark.pp
M manifests/site.pp
M modules/cdh
4 files changed, 45 insertions(+), 13 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/45/209745/1
diff --git a/hieradata/role/common/analytics/spark/standalone.yaml
b/hieradata/role/common/analytics/spark/standalone.yaml
new file mode 100644
index 0000000..e725892
--- /dev/null
+++ b/hieradata/role/common/analytics/spark/standalone.yaml
@@ -0,0 +1,2 @@
+spark_master_host: analytics1003.eqiad.wmnet
+spark_worker_instances: 6
diff --git a/manifests/role/analytics/spark.pp
b/manifests/role/analytics/spark.pp
index d6c750e..26daea8 100644
--- a/manifests/role/analytics/spark.pp
+++ b/manifests/role/analytics/spark.pp
@@ -3,3 +3,32 @@
class role::analytics::spark {
include cdh::spark
}
+
+# == Class role::analytics::spark::standalone
+# Configures a spark standalone cluster.
+# This runs spark daemons outside of YARN.
+# do not include role::analytics::spark
+# and role::analytics::spark::standalone on the same node.
+class role::analytics::spark::standalone {
+ $default_worker_memory = $::realm ? {
+ production => join([$::memorysize_mb - 2048, 'm'], '')
+ default => undef,
+ }
+
+ class { 'cdh::spark':
+ master_host => hiera('spark_master_host', $::fqdn),
+ worker_instances => hiera('spark_worker_instances', undef),
+ worker_cores => hiera('spark_worker_cores',
floor($::processorcount / $spark_worker_instances)),
+ worker_memory => hiera('spark_worker_memory',
$default_worker_memory_mb),
+ }
+}
+
+class role::analytics::spark::standalone::master {
+ require role::analytics::spark::standalone
+ include cdh::spark::master
+}
+
+class role::analytics::spark::standalone::worker {
+ require role::analytics::spark::standalone
+ include cdh::spark::worker
+}
diff --git a/manifests/site.pp b/manifests/site.pp
index 50ef047..e77ac8b 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -60,13 +60,6 @@
}
}
-# analytics1003 is being used for testing kafkatee
-# in preperation for replacing udp2log
-node 'analytics1003.eqiad.wmnet' {
- role analytics
-
- include standard
-}
# analytics1001 is the Hadoop master node:
@@ -86,15 +79,23 @@
include standard
}
-# analytics1004 was previously the Hadoop standby NameNode
-# It is being deprecated.
-node 'analytics1004.eqiad.wmnet' {
+
+
+# NOTE: analytics1003,1004 and 1010 are the remaining analytics Cicsos.
+# They are being used for testing some realtime streaming frameworks.
+
+# analytics1003 is being used for standalone Spark (Streaming).
+# It is the Spark Standalone Master and also a worker.
+node 'analytics1003.eqiad.wmnet' {
+ role analytics::spark::standalone, analytics::spark::standalone::master,
analytics::spark::standalone::worker
+
include standard
}
-# analytics1010 was previously the Hadoop master.
-# It is being deprecated.
-node 'analytics1010.eqiad.wmnet' {
+# analytics1003 and analytics1004 are Spark Standalone workers
+node /analytics10(03|10)].eqiad.wmnet/ {
+ role analytics::spark::standalone, analytics::spark::standalone::worker
+
include standard
}
diff --git a/modules/cdh b/modules/cdh
index fed624a..7c438b4 160000
--- a/modules/cdh
+++ b/modules/cdh
-Subproject commit fed624ae409ad966f7b8dae31e02b879ad38c4b9
+Subproject commit 7c438b46eba77ce15fee2764c0700127fc453181
--
To view, visit https://gerrit.wikimedia.org/r/209745
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Icea3cf8df94fb3164722d5bd05f5b3f89537189a
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <o...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
