Ottomata has uploaded a new change for review.
https://gerrit.wikimedia.org/r/209738
Change subject: Set default spark standalone parameters to undef to use spark's
builtin defaults
......................................................................
Set default spark standalone parameters to undef to use spark's builtin defaults
Change-Id: I91944ff778bd6f78d1ff5464a364382897a67388
---
M manifests/spark.pp
M templates/spark/spark-env.sh.erb
2 files changed, 13 insertions(+), 12 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet/cdh
refs/changes/38/209738/1
diff --git a/manifests/spark.pp b/manifests/spark.pp
index 66df80e..fb8344a 100644
--- a/manifests/spark.pp
+++ b/manifests/spark.pp
@@ -15,26 +15,27 @@
# Default: undef
#
# $worker_cores - Number of cores to allocate per spark worker.
-# This is only used in standalone mode. Default:
$::processorcount
+# This is only used in standalone mode. Default:
undef ($::processorcount)
#
# $worker_memory - Total amount of memory workers are allowed to
use on a node.
-# This is only used in standalone mode. Default:
"${::memorysize_mb - 1024}m",
+# This is only used in standalone mode. Default:
undef ($::memorysize_mb - 1024)
#
# $worker_instances - Number of worker instances to run on a node.
Note that $worker_cores
# will apply to each worker. If you increase
this, make sure to
# make $worker_cores smaller appropriately.
-# This is only used in standalone mode. Default: 1
+# This is only used in standalone mode. Default:
undef (1)
#
# $daemon_memory - Memory to allocate to the Spark master and
worker daemons themselves.
-# This is only used in standalone mode. Default:
512m
+#
+# This is only used in standalone mode. Default:
undef (512m)
#
class cdh::spark(
$hive_support_enabled = true,
$master_host = undef,
- $worker_cores = $::processorcount,
- $worker_memory = "${::memorysize_mb - 1024}m",
- $worker_instances = 1,
- $daemon_memory = '512m',
+ $worker_cores = undef,
+ $worker_memory = undef,
+ $worker_instances = undef,
+ $daemon_memory = undef,
)
{
# Spark requires Hadoop configs installed.
diff --git a/templates/spark/spark-env.sh.erb b/templates/spark/spark-env.sh.erb
index b07cf35..70a4672 100755
--- a/templates/spark/spark-env.sh.erb
+++ b/templates/spark/spark-env.sh.erb
@@ -59,10 +59,10 @@
# if running in Standalone mode, then set these.
if @standalone_enabled
-%>
-export SPARK_WORKER_CORES=<%= @worker_cores %>
-export SPARK_WORKER_MEMORY=<%= @worker_memory %>
-export SPARK_WORKER_INSTANCES=<%= @worker_instances %>
-export SPARK_DAEMON_MEMORY=<%= @daemon_memory %>
+<% if @worker_cores %>export SPARK_WORKER_CORES=<%= @worker_cores %>
+<% if @worker_memory %>export SPARK_WORKER_MEMORY=<%= @worker_memory %><% end
%>
+<% if @worker_instances %>export SPARK_WORKER_INSTANCES=<%= @worker_instances
%><% end %>
+<% if @daemon_memory %>export SPARK_DAEMON_MEMORY=<%= @daemon_memory %><% end
%>
export STANDALONE_SPARK_MASTER_HOST=<%= @standalone_enabled ? @master_host :
'`hostname`' %>
export SPARK_MASTER_IP=$STANDALONE_SPARK_MASTER_HOST
--
To view, visit https://gerrit.wikimedia.org/r/209738
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I91944ff778bd6f78d1ff5464a364382897a67388
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet/cdh
Gerrit-Branch: master
Gerrit-Owner: Ottomata <o...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
