Ottomata has uploaded a new change for review. https://gerrit.wikimedia.org/r/209738
Change subject: Set default spark standalone parameters to undef to use spark's builtin defaults ...................................................................... Set default spark standalone parameters to undef to use spark's builtin defaults Change-Id: I91944ff778bd6f78d1ff5464a364382897a67388 --- M manifests/spark.pp M templates/spark/spark-env.sh.erb 2 files changed, 13 insertions(+), 12 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet/cdh refs/changes/38/209738/1 diff --git a/manifests/spark.pp b/manifests/spark.pp index 66df80e..fb8344a 100644 --- a/manifests/spark.pp +++ b/manifests/spark.pp @@ -15,26 +15,27 @@ # Default: undef # # $worker_cores - Number of cores to allocate per spark worker. -# This is only used in standalone mode. Default: $::processorcount +# This is only used in standalone mode. Default: undef ($::processorcount) # # $worker_memory - Total amount of memory workers are allowed to use on a node. -# This is only used in standalone mode. Default: "${::memorysize_mb - 1024}m", +# This is only used in standalone mode. Default: undef ($::memorysize_mb - 1024) # # $worker_instances - Number of worker instances to run on a node. Note that $worker_cores # will apply to each worker. If you increase this, make sure to # make $worker_cores smaller appropriately. -# This is only used in standalone mode. Default: 1 +# This is only used in standalone mode. Default: undef (1) # # $daemon_memory - Memory to allocate to the Spark master and worker daemons themselves. -# This is only used in standalone mode. Default: 512m +# +# This is only used in standalone mode. Default: undef (512m) # class cdh::spark( $hive_support_enabled = true, $master_host = undef, - $worker_cores = $::processorcount, - $worker_memory = "${::memorysize_mb - 1024}m", - $worker_instances = 1, - $daemon_memory = '512m', + $worker_cores = undef, + $worker_memory = undef, + $worker_instances = undef, + $daemon_memory = undef, ) { # Spark requires Hadoop configs installed. diff --git a/templates/spark/spark-env.sh.erb b/templates/spark/spark-env.sh.erb index b07cf35..70a4672 100755 --- a/templates/spark/spark-env.sh.erb +++ b/templates/spark/spark-env.sh.erb @@ -59,10 +59,10 @@ # if running in Standalone mode, then set these. if @standalone_enabled -%> -export SPARK_WORKER_CORES=<%= @worker_cores %> -export SPARK_WORKER_MEMORY=<%= @worker_memory %> -export SPARK_WORKER_INSTANCES=<%= @worker_instances %> -export SPARK_DAEMON_MEMORY=<%= @daemon_memory %> +<% if @worker_cores %>export SPARK_WORKER_CORES=<%= @worker_cores %> +<% if @worker_memory %>export SPARK_WORKER_MEMORY=<%= @worker_memory %><% end %> +<% if @worker_instances %>export SPARK_WORKER_INSTANCES=<%= @worker_instances %><% end %> +<% if @daemon_memory %>export SPARK_DAEMON_MEMORY=<%= @daemon_memory %><% end %> export STANDALONE_SPARK_MASTER_HOST=<%= @standalone_enabled ? @master_host : '`hostname`' %> export SPARK_MASTER_IP=$STANDALONE_SPARK_MASTER_HOST -- To view, visit https://gerrit.wikimedia.org/r/209738 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I91944ff778bd6f78d1ff5464a364382897a67388 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet/cdh Gerrit-Branch: master Gerrit-Owner: Ottomata <o...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits