Ottomata has submitted this change and it was merged.
Change subject: Be explicit about spark worker memory in hiera
......................................................................
Be explicit about spark worker memory in hiera
Change-Id: I5fb0b2c99517f8e8ebc492e09b197c4957fdaf20
---
M hieradata/role/common/analytics/spark/standalone.yaml
M manifests/role/analytics/spark.pp
2 files changed, 4 insertions(+), 6 deletions(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
diff --git a/hieradata/role/common/analytics/spark/standalone.yaml
b/hieradata/role/common/analytics/spark/standalone.yaml
index e725892..b52f478 100644
--- a/hieradata/role/common/analytics/spark/standalone.yaml
+++ b/hieradata/role/common/analytics/spark/standalone.yaml
@@ -1,2 +1,5 @@
spark_master_host: analytics1003.eqiad.wmnet
+# Ciscos have 24(ish) processors, use 6 workers each with 4 processes.
spark_worker_instances: 6
+# Ciscos have a lot of memory. Save 2560m for OS and daemons
+spark_worker_memory: 190825
diff --git a/manifests/role/analytics/spark.pp
b/manifests/role/analytics/spark.pp
index 6ce814e..9411e4f 100644
--- a/manifests/role/analytics/spark.pp
+++ b/manifests/role/analytics/spark.pp
@@ -10,16 +10,11 @@
# do not include role::analytics::spark
# and role::analytics::spark::standalone on the same node.
class role::analytics::spark::standalone {
- $default_worker_memory = $::realm ? {
- production => join([$::memorysize_mb - 2048, 'm'], ''),
- default => undef,
- }
-
class { 'cdh::spark':
master_host => hiera('spark_master_host', $::fqdn),
worker_instances => hiera('spark_worker_instances', undef),
worker_cores => hiera('spark_worker_cores',
floor($::processorcount / hiera('spark_worker_instances', 1))),
- worker_memory => hiera('spark_worker_memory',
$default_worker_memory_mb),
+ worker_memory => hiera('spark_worker_memory', undef)
}
}
--
To view, visit https://gerrit.wikimedia.org/r/209759
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I5fb0b2c99517f8e8ebc492e09b197c4957fdaf20
Gerrit-PatchSet: 2
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <o...@wikimedia.org>
Gerrit-Reviewer: Ottomata <o...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
