Ottomata has uploaded a new change for review.
https://gerrit.wikimedia.org/r/148143
Change subject: Update for cdh module, move fair-schedule.xml.erb queue
allocation to puppet repo
......................................................................
Update for cdh module, move fair-schedule.xml.erb queue allocation to puppet
repo
Remove unused hadoop template file
Change-Id: I4d2f4aefae282fae0d23efb0cd6fcee1ff4ed33f
---
M manifests/role/analytics/hadoop.pp
M modules/cdh
D templates/hadoop/fair-scheduler-allocation.xml.erb
M templates/hadoop/fair-scheduler.xml.erb
4 files changed, 22 insertions(+), 35 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/43/148143/1
diff --git a/manifests/role/analytics/hadoop.pp
b/manifests/role/analytics/hadoop.pp
index 1b007ee..3142876 100644
--- a/manifests/role/analytics/hadoop.pp
+++ b/manifests/role/analytics/hadoop.pp
@@ -54,6 +54,7 @@
$mapreduce_output_compression_codec =
'org.apache.hadoop.io.compress.SnappyCodec'
$mapreduce_output_compression_type = 'BLOCK'
$mapreduce_job_reuse_jvm_num_tasks = 1
+ $fair_scheduler_template = 'hadoop/fair-scheduler.xml.erb'
# Configs specific to Production.
@@ -232,8 +233,10 @@
mapreduce_task_io_sort_mb => $mapreduce_task_io_sort_mb,
mapreduce_task_io_sort_factor =>
$mapreduce_task_io_sort_factor,
yarn_nodemanager_resource_memory_mb =>
$yarn_nodemanager_resource_memory_mb,
- # use net-topology.py.erb to map hostname to /datacenter/rack/row id.
+ # Use net-topology.py.erb to map hostname to /datacenter/rack/row id.
net_topology_script_template =>
$net_topology_script_template,
+ # Use fair-scheduler.xml.erb to define FairScheduler queues.
+ fair_scheduler_template => $fair_scheduler_template,
}
# If in production AND the current node is a journalnode, then
diff --git a/modules/cdh b/modules/cdh
index 69b6d3e..1a62a58 160000
--- a/modules/cdh
+++ b/modules/cdh
-Subproject commit 69b6d3e853d248c5977a6909eaceab72a9620284
+Subproject commit 1a62a58c1f7cc80e221789fe8f9a6c9fb18109f7
diff --git a/templates/hadoop/fair-scheduler-allocation.xml.erb
b/templates/hadoop/fair-scheduler-allocation.xml.erb
deleted file mode 100644
index c6ab343..0000000
--- a/templates/hadoop/fair-scheduler-allocation.xml.erb
+++ /dev/null
@@ -1,22 +0,0 @@
-<?xml version="1.0"?>
-<allocations>
- <fairSharePreemptionTimeout>30</fairSharePreemptionTimeout>
-
- <queue name="default">
- <maxRunningApps>50</maxRunningApps>
- <schedulingMode>fair</schedulingMode>
- </queue>
-
- <queue name="standard">
- <!-- <weight>2.0</weight> -->
- <aclSubmitApps>hdfs,stats</aclSubmitApps>
- <maxRunningApps>50</maxRunningApps>
- <schedulingMode>fair</schedulingMode>
- </queue>
-
- <queue name="adhoc">
- <maxRunningApps>10</maxRunningApps>
- <schedulingMode>fair</schedulingMode>
- </queue>
-
-</allocations>
diff --git a/templates/hadoop/fair-scheduler.xml.erb
b/templates/hadoop/fair-scheduler.xml.erb
index 689716c..74fc1d3 100644
--- a/templates/hadoop/fair-scheduler.xml.erb
+++ b/templates/hadoop/fair-scheduler.xml.erb
@@ -1,16 +1,22 @@
<?xml version="1.0"?>
-<configuration>
+<allocations>
+ <fairSharePreemptionTimeout>30</fairSharePreemptionTimeout>
- <property>
- <name>yarn.scheduler.fair.allocation.file</name>
- <value>/etc/hadoop/conf/fair-scheduler-allocation.xml</value>
- <description>Path to allocation file. An allocation file is an XML
manifest describing queues and their properties, in addition to certain policy
defaults. This file must be in XML format as described in the next
section.</description>
- </property>
+ <queue name="default">
+ <maxRunningApps>50</maxRunningApps>
+ <schedulingMode>fair</schedulingMode>
+ </queue>
- <property>
- <name>yarn.scheduler.fair.user-as-default-queue</name>
- <value>false</value>
- <description>Whether to use the username associated with the allocation as
the default queue name, in the event that a queue name is not specified. If
this is set to "false" or unset, all jobs have a shared default queue, called
"default". Defaults to true.</description>
- </property>
+ <queue name="standard">
+ <!-- <weight>2.0</weight> -->
+ <aclSubmitApps>hdfs,stats</aclSubmitApps>
+ <maxRunningApps>50</maxRunningApps>
+ <schedulingMode>fair</schedulingMode>
+ </queue>
-</configuration>
+ <queue name="adhoc">
+ <maxRunningApps>10</maxRunningApps>
+ <schedulingMode>fair</schedulingMode>
+ </queue>
+
+</allocations>
--
To view, visit https://gerrit.wikimedia.org/r/148143
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I4d2f4aefae282fae0d23efb0cd6fcee1ff4ed33f
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
