Ottomata has submitted this change and it was merged.
Change subject: Moving compression configs to mapred-site.xml where they belong
......................................................................
Moving compression configs to mapred-site.xml where they belong
Change-Id: I4adbf4ca20a194f9beaa8a7fb407c151e6809a0f
---
M manifests/hadoop.pp
M manifests/hadoop/defaults.pp
M templates/hadoop/mapred-site.xml.erb
M templates/hadoop/yarn-site.xml.erb
4 files changed, 36 insertions(+), 36 deletions(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
jenkins-bot: Verified
diff --git a/manifests/hadoop.pp b/manifests/hadoop.pp
index 57f26ed..49562a0 100644
--- a/manifests/hadoop.pp
+++ b/manifests/hadoop.pp
@@ -21,8 +21,6 @@
# $yarn_local_path - path relative to JBOD mount point for yarn local
directories.
# $yarn_logs_path - path relative to JBOD mount point for yarn log
directories.
# $dfs_block_size - HDFS block size in bytes. Default 64MB.
-# $enable_intermediate_compression - If true, intermediate MapReduce data
will be compressed with Snappy.
-# $enable_final_compession - If true, Final output of MapReduce
jobs will be compressed with Snappy.
# $io_file_buffer_size
# $map_tasks_maximum
# $reduce_tasks_maximum
@@ -34,6 +32,8 @@
# $mapreduce_task_io_sort_factor
# $mapreduce_map_java_opts
# $mapreduce_child_java_opts
+# $mapreduce_intermediate_compression - If true, intermediate MapReduce
data will be compressed with Snappy. Default: true.
+# $mapreduce_final_compession - If true, Final output of MapReduce
jobs will be compressed with Snappy. Default: false.
# $yarn_nodemanager_resource_memory_mb
# $yarn_resourcemanager_scheduler_class - If you change this (e.g. to
FairScheduler), you should also provide your own scheduler config .xml files
outside of the cdh4 module.
# $use_yarn
@@ -49,8 +49,6 @@
$dfs_block_size =
$::cdh4::hadoop::defaults::dfs_block_size,
$enable_jmxremote =
$::cdh4::hadoop::defaults::enable_jmxremote,
$enable_webhdfs =
$::cdh4::hadoop::defaults::enable_webhdfs,
- $enable_intermediate_compression =
$::cdh4::hadoop::defaults::enable_intermediate_compression,
- $enable_final_compession =
$::cdh4::hadoop::defaults::enable_final_compession,
$io_file_buffer_size =
$::cdh4::hadoop::defaults::io_file_buffer_size,
$mapreduce_map_tasks_maximum =
$::cdh4::hadoop::defaults::mapreduce_map_tasks_maximum,
$mapreduce_reduce_tasks_maximum =
$::cdh4::hadoop::defaults::mapreduce_reduce_tasks_maximum,
@@ -62,6 +60,8 @@
$mapreduce_task_io_sort_factor =
$::cdh4::hadoop::defaults::mapreduce_task_io_sort_factor,
$mapreduce_map_java_opts =
$::cdh4::hadoop::defaults::mapreduce_map_java_opts,
$mapreduce_reduce_java_opts =
$::cdh4::hadoop::defaults::mapreduce_reduce_java_opts,
+ $mapreduce_intermediate_compression =
$::cdh4::hadoop::defaults::mapreduce_intermediate_compression,
+ $mapreduce_final_compession =
$::cdh4::hadoop::defaults::mapreduce_final_compession,
$yarn_nodemanager_resource_memory_mb =
$::cdh4::hadoop::defaults::yarn_nodemanager_resource_memory_mb,
$yarn_resourcemanager_scheduler_class =
$::cdh4::hadoop::defaults::yarn_resourcemanager_scheduler_class,
$use_yarn =
$::cdh4::hadoop::defaults::use_yarn
diff --git a/manifests/hadoop/defaults.pp b/manifests/hadoop/defaults.pp
index 5ad57aa..3fc6c5d 100644
--- a/manifests/hadoop/defaults.pp
+++ b/manifests/hadoop/defaults.pp
@@ -10,8 +10,6 @@
$dfs_block_size = 67108864 # 64MB default
$enable_jmxremote = true
$enable_webhdfs = true
- $enable_intermediate_compression = true
- $enable_final_compession = false
$io_file_buffer_size = undef
$mapreduce_map_tasks_maximum = undef
$mapreduce_reduce_tasks_maximum = undef
@@ -23,6 +21,8 @@
$mapreduce_task_io_sort_factor = undef
$mapreduce_map_java_opts = undef
$mapreduce_reduce_java_opts = undef
+ $mapreduce_intermediate_compression = true
+ $mapreduce_final_compression = false
$yarn_nodemanager_resource_memory_mb = undef
$yarn_resourcemanager_scheduler_class = undef
$use_yarn = true
diff --git a/templates/hadoop/mapred-site.xml.erb
b/templates/hadoop/mapred-site.xml.erb
index 137c43a..8864b15 100644
--- a/templates/hadoop/mapred-site.xml.erb
+++ b/templates/hadoop/mapred-site.xml.erb
@@ -31,7 +31,7 @@
<name>mapreduce.jobhistory.done-dir</name>
<value>/user/history/done</value>
</property>
-
+
<property>
<name>mapred.job.tracker</name>
<!-- Hue gets confused if mapred.job.tracker is not set,
@@ -115,6 +115,30 @@
</property>
<% end -%>
+ <!-- MapReduce intermediate compression -->
+ <property>
+ <name>mapreduce.map.output.compress</name>
+ <value><%= mapreduce_intermediate_compression %></value>
+ </property>
+ <property>
+ <name>mapreduce.map.output.compress.codec</name>
+ <value>org.apache.hadoop.io.compress.SnappyCodec</value>
+ </property>
+
+ <!-- MapReduce final output compression -->
+ <property>
+ <name>mapreduce.output.fileoutputformat.compress</name>
+ <value><%= mapreduce_final_compression %></value>
+ </property>
+ <property>
+ <name>mapred.output.fileoutputformat.compress.codec</name>
+ <value>org.apache.hadoop.io.compress.SnappyCodec</value>
+ </property>
+ <property>
+ <name>mapreduce.output.fileoutputformat.compress.type</name>
+ <value>BLOCK</value>
+ </property>
+
<property>
<name>mapred.hosts.exclude</name>
<value><%= config_directory %>/hosts.exclude</value>
diff --git a/templates/hadoop/yarn-site.xml.erb
b/templates/hadoop/yarn-site.xml.erb
index c11b8f2..3d51195 100644
--- a/templates/hadoop/yarn-site.xml.erb
+++ b/templates/hadoop/yarn-site.xml.erb
@@ -58,10 +58,6 @@
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
- <property>
- <name>yarn.log-aggregation-enable</name>
- <value>true</value>
- </property>
<% if @datanode_mounts -%>
<property>
@@ -85,6 +81,11 @@
<% end -%>
<property>
+ <name>yarn.log-aggregation-enable</name>
+ <value>true</value>
+ </property>
+
+ <property>
<description>Where to aggregate logs to.</description>
<name>yarn.nodemanager.remote-app-log-dir</name>
<value>/var/log/hadoop-yarn/apps</value>
@@ -96,17 +97,6 @@
<value>/user</value>
</property>
-
- <!-- MapReduce intermediate compression -->
- <property>
- <name>mapreduce.map.output.compress</name>
- <value><%= enable_intermediate_compression %></value>
- </property>
- <property>
- <name>mapreduce.map.output.compress.codec</name>
- <value>org.apache.hadoop.io.compress.SnappyCodec</value>
- </property>
-
<property>
<name>yarn.resourcemanager.nodes.exclude-path</name>
<value><%= config_directory %>/hosts.exclude</value>
@@ -115,20 +105,6 @@
This is useful for decommissioning nodes.
</description>
</property>
-
- <!-- MapReduce final output compression -->
- <property>
- <name>mapreduce.output.fileoutputformat.compress</name>
- <value><%= enable_final_compession %></value>
- </property>
- <property>
- <name>mapred.output.fileoutputformat.compress.codec</name>
- <value>org.apache.hadoop.io.compress.SnappyCodec</value>
- </property>
- <property>
- <name>mapreduce.output.fileoutputformat.compress.type</name>
- <value>BLOCK</value>
- </property>
<property>
<description>Classpath for typical applications.</description>
--
To view, visit https://gerrit.wikimedia.org/r/66869
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I4adbf4ca20a194f9beaa8a7fb407c151e6809a0f
Gerrit-PatchSet: 2
Gerrit-Project: operations/puppet/cdh4
Gerrit-Branch: master
Gerrit-Owner: Ottomata <o...@wikimedia.org>
Gerrit-Reviewer: Ottomata <o...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
