Author: rvs
Date: Wed Nov 30 23:31:47 2011
New Revision: 1208881
URL: http://svn.apache.org/viewvc?rev=1208881&view=rev
Log:
BIGTOP-273. Need to update puppet recipies for Hadoop 0.23 deployment
Added:
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/cluster.pp
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hadoop
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/taskcontroller.cfg
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/cluster.pp
URL:
http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/cluster.pp?rev=1208881&r1=1208880&r2=1208881&view=diff
==============================================================================
---
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/cluster.pp
(original)
+++
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/cluster.pp
Wed Nov 30 23:31:47 2011
@@ -20,6 +20,12 @@ class hadoop_cluster_node {
# $hadoop_dfs_namenode_plugins="org.apache.hadoop.thriftfs.NamenodePlugin"
# $hadoop_dfs_datanode_plugins="org.apache.hadoop.thriftfs.DatanodePlugin"
+ $hadoop_rm_host="$hadoop_head_node"
+ $hadoop_rt_port="8025"
+ $hadoop_rm_port="8040"
+ $hadoop_sc_port="8030"
+ $hadoop_rt_thrift_port="9290"
+
$hadoop_jobtracker_host="$hadoop_head_node"
$hadoop_jobtracker_port="8021"
$hadoop_jobtracker_thrift_port="9290"
@@ -35,7 +41,8 @@ class hadoop_cluster_node {
$namenode_data_dirs = ["/mnt/namenode"]
$hdfs_data_dirs = ["/mnt/hdfs"]
- $mapred_data_dirs = ["/mnt/scratch"]
+ $mapred_data_dirs = ["/mnt/mapred"]
+ $yarn_data_dirs = ["/mnt/yarn"]
if ($hadoop_security_authentication == "kerberos") {
$kerberos_domain = "compute-1.internal"
@@ -60,12 +67,11 @@ class hadoop_worker_node inherits hadoop
auth => $hadoop_security_authentication,
}
- hadoop::tasktracker { "tasktracker":
- namenode_host => $hadoop_namenode_host,
- namenode_port => $hadoop_namenode_port,
- jobtracker_host => $hadoop_jobtracker_host,
- jobtracker_port => $hadoop_jobtracker_port,
- dirs => $mapred_data_dirs,
+ hadoop::nodemanager { "nodemanager":
+ rm_host => $hadoop_rm_host,
+ rm_port => $hadoop_rm_port,
+ rt_port => $hadoop_rt_port,
+ dirs => $yarn_data_dirs,
auth => $hadoop_security_authentication,
}
@@ -74,6 +80,15 @@ class hadoop_worker_node inherits hadoop
zookeeper_quorum => $hadoop_hbase_zookeeper_quorum,
kerberos_realm => $kerberos_realm,
}
+
+ hadoop::mapred-app { "mapred-app":
+ namenode_host => $hadoop_namenode_host,
+ namenode_port => $hadoop_namenode_port,
+ jobtracker_host => $hadoop_jobtracker_host,
+ jobtracker_port => $hadoop_jobtracker_port,
+ auth => $hadoop_security_authentication,
+ dirs => $mapred_data_dirs,
+ }
}
class hadoop_head_node inherits hadoop_cluster_node {
@@ -83,9 +98,8 @@ class hadoop_head_node inherits hadoop_c
}
hadoop::namenode { "namenode":
+ host => $hadoop_namenode_host,
port => $hadoop_namenode_port,
- jobtracker_host => $hadoop_jobtracker_host,
- jobtracker_port => $hadoop_jobtracker_port,
dirs => $namenode_data_dirs,
# thrift_port => $hadoop_namenode_thrift_port,
auth => $hadoop_security_authentication,
@@ -97,12 +111,11 @@ class hadoop_head_node inherits hadoop_c
auth => $hadoop_security_authentication,
}
- hadoop::jobtracker { "jobtracker":
- namenode_host => $hadoop_namenode_host,
- namenode_port => $hadoop_namenode_port,
- host => $hadoop_jobtracker_host,
- port => $hadoop_jobtracker_port,
- dirs => $mapred_data_dirs,
+ hadoop::resourcemanager { "resourcemanager":
+ host => $hadoop_rm_host,
+ port => $hadoop_rm_port,
+ rt_port => $hadoop_rt_port,
+ sc_port => $hadoop_sc_port,
# thrift_port => $hadoop_jobtracker_thrift_port,
auth => $hadoop_security_authentication,
}
@@ -122,7 +135,7 @@ class hadoop_head_node inherits hadoop_c
ensemble => $hadoop_zookeeper_ensemble,
}
- hadoop::create_hdfs_dirs { [ "/mapred", "/tmp", "/system", "/user",
"/hbase", "/benchmarks", "/user/jenkins", "/user/hive" ]:
+ hadoop::create_hdfs_dirs { [ "/mapred", "/tmp", "/system", "/user",
"/hbase", "/benchmarks", "/user/jenkins", "/user/hive", "/user/root",
"/user/history" ]:
hdfs_dirs_meta => { "/tmp" => { perm => "777", user => "hdfs" },
"/mapred" => { perm => "755", user => "mapred" },
"/system" => { perm => "755", user => "hdfs" },
@@ -130,19 +143,20 @@ class hadoop_head_node inherits hadoop_c
"/hbase" => { perm => "755", user => "hbase" },
"/benchmarks" => { perm => "777", user => "hdfs" },
"/user/jenkins" => { perm => "777", user => "jenkins"},
+ "/user/history" => { perm => "777", user => "mapred" },
+ "/user/root" => { perm => "777", user => "root" },
"/user/hive" => { perm => "777", user => "hive" }
},
}
}
class hadoop_gateway_node inherits hadoop_head_node {
- # hadoop::client { "gateway":
- # namenode_host => $hadoop_namenode_host,
- # namenode_port => $hadoop_namenode_port,
- # jobtracker_host => $hadoop_jobtracker_host,
- # jobtracker_port => $hadoop_jobtracker_port,
- # # auth => $hadoop_security_authentication,
- # }
-
+ hadoop::client { "hadoop client":
+ namenode_host => $hadoop_namenode_host,
+ namenode_port => $hadoop_namenode_port,
+ jobtracker_host => $hadoop_jobtracker_host,
+ jobtracker_port => $hadoop_jobtracker_port,
+ # auth => $hadoop_security_authentication,
+ }
mahout::client { "mahout client":
}
hadoop-pig::client { "pig client":
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
URL:
http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp?rev=1208881&r1=1208880&r2=1208881&view=diff
==============================================================================
---
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
(original)
+++
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
Wed Nov 30 23:31:47 2011
@@ -21,46 +21,63 @@ class hadoop {
*/
class common {
file {
- "/etc/hadoop/conf/core-site.xml":
- content => template('hadoop/core-site.xml'),
+ "/etc/hadoop/conf/hadoop-env.sh":
+ content => template('hadoop/hadoop-env.sh'),
require => [Package["hadoop"]],
}
file {
- "/etc/hadoop/conf/mapred-site.xml":
- content => template('hadoop/mapred-site.xml'),
+ "/etc/default/hadoop":
+ content => template('hadoop/hadoop'),
require => [Package["hadoop"]],
}
+ package { "hadoop":
+ ensure => latest,
+ require => Package["jdk"],
+ }
+
+ #FIXME: package { "hadoop-native":
+ # ensure => latest,
+ # require => [Package["hadoop"]],
+ #}
+ }
+
+ class common-yarn inherits common {
file {
- "/etc/hadoop/conf/hdfs-site.xml":
- content => template('hadoop/hdfs-site.xml'),
+ "/etc/yarn/conf/yarn-site.xml":
+ content => template('hadoop/yarn-site.xml'),
require => [Package["hadoop"]],
}
+ }
+ class common-hdfs inherits common {
file {
- "/etc/hadoop/conf/hadoop-env.sh":
- content => template('hadoop/hadoop-env.sh'),
+ "/etc/hadoop/conf/core-site.xml":
+ content => template('hadoop/core-site.xml'),
require => [Package["hadoop"]],
}
file {
- "/etc/default/hadoop":
- content => template('hadoop/hadoop'),
+ "/etc/hadoop/conf/hdfs-site.xml":
+ content => template('hadoop/hdfs-site.xml'),
require => [Package["hadoop"]],
}
+ }
- package { "hadoop":
- ensure => latest,
+ class common-mapred-app inherits common-hdfs {
+ file {
+ "/etc/hadoop/conf/mapred-site.xml":
+ content => template('hadoop/mapred-site.xml'),
+ require => [Package["hadoop"]],
}
- package { "hadoop-native":
- ensure => latest,
+ file { "/etc/hadoop/conf/taskcontroller.cfg":
+ content => template('hadoop/taskcontroller.cfg'),
require => [Package["hadoop"]],
}
}
-
define datanode ($namenode_host, $namenode_port, $port = "50075", $auth =
"simple", $dirs = ["/tmp/data"]) {
$hadoop_namenode_host = $namenode_host
@@ -68,7 +85,7 @@ class hadoop {
$hadoop_datanode_port = $port
$hadoop_security_authentication = $auth
- include common
+ include common-hdfs
package { "hadoop-datanode":
ensure => latest,
@@ -76,10 +93,10 @@ class hadoop {
}
if ($hadoop_security_authentication == "kerberos") {
- package { "hadoop-sbin":
- ensure => latest,
- require => [Package["hadoop"]],
- }
+ #FIXME: package { "hadoop-sbin":
+ # ensure => latest,
+ # require => [Package["hadoop"]],
+ #}
}
service { "hadoop-datanode":
@@ -108,16 +125,14 @@ class hadoop {
}
}
- define namenode ($jobtracker_host, $jobtracker_port, $host = $fqdn , $port =
"8020", $thrift_port= "10090", $auth = "simple", $dirs = ["/tmp/nn"]) {
+ define namenode ($host = $fqdn , $port = "8020", $thrift_port= "10090",
$auth = "simple", $dirs = ["/tmp/nn"]) {
$hadoop_namenode_host = $host
$hadoop_namenode_port = $port
$hadoop_namenode_thrift_port = $thrift_port
- $hadoop_jobtracker_host = $jobtracker_host
- $hadoop_jobtracker_port = $jobtracker_port
$hadoop_security_authentication = $auth
- include common
+ include common-hdfs
package { "hadoop-namenode":
ensure => latest,
@@ -127,14 +142,14 @@ class hadoop {
service { "hadoop-namenode":
ensure => running,
hasstatus => true,
- subscribe => [Package["hadoop-namenode"],
File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
+ subscribe => [Package["hadoop-namenode"],
File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"],
File["/etc/hadoop/conf/hadoop-env.sh"]],
require => [Package["hadoop-namenode"], Exec["namenode format"]],
}
exec { "namenode format":
user => "hdfs",
command => "/bin/bash -c 'yes Y | hadoop namenode -format >>
/tmp/nn.format.log 2>&1'",
- creates => inline_template("<%= hadoop_storage_locations.split(';')[0]
%>/namenode/image"),
+ creates => inline_template("<%= hadoop_storage_locations.split(';')[0]
%>/namenode/current/VERSION"),
require => [Package["hadoop-namenode"]],
} <- file { $dirs:
ensure => directory,
@@ -145,88 +160,91 @@ class hadoop {
}
}
+ define secondarynamenode ($namenode_host, $namenode_port, $port = "50090",
$auth = "simple") {
- define jobtracker ($namenode_host, $namenode_port, $host = $fqdn, $port =
"8021", $thrift_port = "9290", $auth = "simple", $dirs = ["/tmp/mr"]) {
-
- $hadoop_namenode_host = $namenode_host
- $hadoop_namenode_port = $namenode_port
- $hadoop_jobtracker_thrift_port = $thrift_port
- $hadoop_jobtracker_host = $host
- $hadoop_jobtracker_port = $port
- $hadoop_security_authentication = $auth
+ $hadoop_secondarynamenode_port = $port
+ $hadoop_security_authentication = $auth
- include common
+ include common-hdfs
- package { "hadoop-jobtracker":
- ensure => latest,
- require => Package["jdk"],
- }
+ package { "hadoop-secondarynamenode":
+ ensure => latest,
+ require => Package["jdk"],
+ }
- service { "hadoop-jobtracker":
+ service { "hadoop-secondarynamenode":
ensure => running,
hasstatus => true,
- subscribe => [Package["hadoop-jobtracker"],
File["/etc/hadoop/conf/core-site.xml"],
File["/etc/hadoop/conf/mapred-site.xml"],
File["/etc/hadoop/conf/hadoop-env.sh"]],
- require => [ Package["hadoop-jobtracker"] ]
- } <- file { $dirs:
- ensure => directory,
- owner => mapred,
- group => mapred,
- mode => 755,
- require => [Package["hadoop"]],
+ subscribe => [Package["hadoop-secondarynamenode"],
File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"],
File["/etc/hadoop/conf/hadoop-env.sh"]],
+ require => [Package["hadoop-secondarynamenode"]],
}
}
- define tasktracker ($namenode_host, $namenode_port, $jobtracker_host,
$jobtracker_port, $auth = "simple", $dirs = ["/tmp/mr"]){
+ define resourcemanager ($host = $fqdn, $port = "8040", $rt_port = "8025",
$sc_port = "8030", $thrift_port = "9290", $auth = "simple") {
+ $hadoop_rm_host = $host
+ $hadoop_rm_port = $port
+ $hadoop_rt_port = $rt_port
+ $hadoop_sc_port = $sc_port
+ $hadoop_security_authentication = $auth
- $hadoop_namenode_host = $namenode_host
- $hadoop_namenode_port = $namenode_port
- $hadoop_jobtracker_host = $jobtracker_host
- $hadoop_jobtracker_port = $jobtracker_port
- $hadoop_security_authentication = $auth
+ include common-yarn
- include common
+ package { "hadoop-resourcemanager":
+ ensure => latest,
+ require => Package["jdk"],
+ }
- package { "hadoop-tasktracker":
- ensure => latest,
- require => Package["jdk"],
- }
-
- file { "/etc/hadoop/conf/taskcontroller.cfg":
- content => template('hadoop/taskcontroller.cfg'),
+ service { "hadoop-resourcemanager":
+ ensure => running,
+ hasstatus => true,
+ subscribe => [Package["hadoop-resourcemanager"],
File["/etc/hadoop/conf/hadoop-env.sh"], File["/etc/yarn/conf/yarn-site.xml"]],
+ require => [ Package["hadoop-resourcemanager"] ]
}
+ }
+
+
+ define nodemanager ($rm_host, $rm_port, $rt_port, $auth = "simple", $dirs =
["/tmp/yarn"]){
+ $hadoop_rm_host = $rm_host
+ $hadoop_rm_port = $rm_port
+ $hadoop_rt_port = $rt_port
- service { "hadoop-tasktracker":
+ include common-yarn
+
+ package { "hadoop-nodemanager":
+ ensure => latest,
+ require => Package["jdk"],
+ }
+
+ service { "hadoop-nodemanager":
ensure => running,
hasstatus => true,
- subscribe => [Package["hadoop-tasktracker"],
File["/etc/hadoop/conf/core-site.xml"],
File["/etc/hadoop/conf/mapred-site.xml"],
File["/etc/hadoop/conf/hadoop-env.sh"]],
- require => [ Package["hadoop-tasktracker"],
File["/etc/hadoop/conf/taskcontroller.cfg"] ],
+ subscribe => [Package["hadoop-nodemanager"],
File["/etc/hadoop/conf/hadoop-env.sh"], File["/etc/yarn/conf/yarn-site.xml"]],
+ require => [ Package["hadoop-nodemanager"] ],
} <- file { $dirs:
ensure => directory,
- owner => mapred,
- group => mapred,
+ owner => yarn,
+ group => yarn,
mode => 755,
require => [Package["hadoop"]],
}
}
-
- define secondarynamenode ($namenode_host, $namenode_port, $port = "50090",
$auth = "simple") {
-
- $hadoop_secondarynamenode_port = $port
+ define mapred-app ($namenode_host, $namenode_port, $jobtracker_host,
$jobtracker_port, $auth = "simple", $dirs = ["/tmp/mr"]){
+ $hadoop_namenode_host = $namenode_host
+ $hadoop_namenode_port = $namenode_port
+ $hadoop_jobtracker_host = $jobtracker_host
+ $hadoop_jobtracker_port = $jobtracker_port
$hadoop_security_authentication = $auth
- include common
- package { "hadoop-secondarynamenode":
- ensure => latest,
- require => Package["jdk"],
- }
+ include common-mapred-app
- service { "hadoop-secondarynamenode":
- ensure => running,
- hasstatus => true,
- subscribe => [Package["hadoop-secondarynamenode"],
File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
- require => [Package["hadoop-secondarynamenode"]],
+ file { $dirs:
+ ensure => directory,
+ owner => mapred,
+ group => mapred,
+ mode => 755,
+ require => [Package["hadoop"]],
}
}
@@ -237,10 +255,10 @@ class hadoop {
$hadoop_jobtracker_port = $jobtracker_port
$hadoop_security_authentication = $auth
- include common
+ include common-mapred-app
- package { ["hadoop-doc", "hadoop-source", "hadoop-debuginfo",
- "hadoop-fuse", "hadoop-libhdfs", "hadoop-pipes"]:
+ # FIXME: "hadoop-source", "hadoop-fuse", "hadoop-pipes"
+ package { ["hadoop-doc", "hadoop-debuginfo", "hadoop-libhdfs"]:
ensure => latest,
require => [Package["jdk"], Package["hadoop"]],
}
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hadoop
URL:
http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hadoop?rev=1208881&r1=1208880&r2=1208881&view=diff
==============================================================================
---
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hadoop
(original)
+++
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hadoop
Wed Nov 30 23:31:47 2011
@@ -12,9 +12,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
export HADOOP_HOME_WARN_SUPPRESS=true
-export HADOOP_HOME=/usr/lib/hadoop
-export HADOOP_PID_DIR=/var/run/hadoop
-export HADOOP_LOG_DIR=/var/log/hadoop
+export HADOOP_PREFIX=/usr/lib/hadoop
+export HADOOP_LIBEXEC_DIR=/usr/libexec
export HADOOP_NAMENODE_USER=hdfs
export HADOOP_SECONDARYNAMENODE_USER=hdfs
export HADOOP_DATANODE_USER=hdfs
@@ -25,4 +24,13 @@ export HADOOP_SECURE_DN_LOG_DIR=$HADOOP_
<% end %>
export HADOOP_JOBTRACKER_USER=mapred
export HADOOP_TASKTRACKER_USER=mapred
-export HADOOP_IDENT_STRING=hadoop
+export HADOOP_IDENT_STRING=hdfs
+export HADOOP_PID_DIR=/var/run/hadoop
+export HADOOP_LOG_DIR=/var/log/hadoop
+export HADOOP_CONF_DIR=/etc/hadoop/conf
+export YARN_CONF_DIR=/etc/yarn/conf
+export
HADOOP_CLASSPATH="${HADOOP_CLASSPATH}:${HADOOP_PREFIX}/*:${HADOOP_PREFIX}/lib/*"
+
+# Pointless definitions making yarn happy
+export HADOOP_COMMON_HOME=${HADOOP_PREFIX}
+export HADOOP_HDFS_HOME=${HADOOP_PREFIX}
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
URL:
http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml?rev=1208881&r1=1208880&r2=1208881&view=diff
==============================================================================
---
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
(original)
+++
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
Wed Nov 30 23:31:47 2011
@@ -116,6 +116,12 @@
<value><% hadoop_storage_locations.split(";").each do |storage_location|
%><%= storage_location%>/namenode,<% end %></value>
</property>
+ <property>
+ <name>dfs.permissions.superusergroup</name>
+ <value>hadoop</value>
+ <description>The name of the group of super-users.</description>
+ </property>
+
<!-- Enable Hue plugins -->
<% if has_variable?("hadoop_dfs_namenode_plugins") %>
<property>
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
URL:
http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml?rev=1208881&r1=1208880&r2=1208881&view=diff
==============================================================================
---
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
(original)
+++
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
Wed Nov 30 23:31:47 2011
@@ -49,7 +49,7 @@
<!-- TaskController settings -->
<property>
- <name>mapred.task.tracker.task-controller</name>
+ <name>mapreduce.tasktracker.taskcontroller</name>
<value>org.apache.hadoop.mapred.LinuxTaskController</value>
</property>
<property>
@@ -60,7 +60,7 @@
<% if has_variable?("mapred_acls_enabled") %>
<property>
- <name>mapred.acls.enabled</name>
+ <name>mapreduce.cluster.acls.enabled</name>
<value><%= mapred_acls_enabled %></value>
</property>
<% end %>
@@ -68,7 +68,7 @@
<!-- specify JobTracker TaskScheduler -->
<% if has_variable?("hadoop_jobtracker_taskscheduler") %>
<property>
- <name>mapred.jobtracker.taskScheduler</name>
+ <name>mapreduce.jobtracker.taskscheduler</name>
<value><%= hadoop_jobtracker_taskscheduler %></value>
</property>
<% end %>
@@ -95,19 +95,34 @@
<% end %>
<property>
- <name>mapred.job.tracker</name>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
+ </property>
+
+ <property>
+ <name>mapreduce.admin.user.env</name>
+
<value>CLASSPATH=/etc/hadoop/conf/*:/usr/lib/hadoop/*:/usr/lib/hadoop/lib/*:/usr/lib/hadoop/lib/native/*</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobtracker.address</name>
<value><%= hadoop_jobtracker_host %>:<%= hadoop_jobtracker_port%></value>
</property>
<property>
- <name>mapred.local.dir</name>
- <value><% hadoop_storage_locations.split(";").each do |storage_location|
%><%= storage_location%>/scratch,<% end %></value>
+ <name>mapreduce.cluster.local.dir</name>
+ <value><% hadoop_storage_locations.split(";").each do |storage_location|
%><%= storage_location%>/mapred,<% end %></value>
<final>true</final>
</property>
+ <!-- property>
+ <name>mapreduce.cluster.temp.dir</name>
+ <value>/mapred/system</value>
+ </property -->
+
<property>
- <name>mapred.child.java.opts</name>
- <value>-Xmx1024m</value>
+ <name>mapreduce.jobtracker.system.dir</name>
+ <value>/mapred/system</value>
</property>
<property>
@@ -116,8 +131,13 @@
</property>
<property>
- <name>mapred.system.dir</name>
- <value>/mapred/system</value>
+ <name>yarn.app.mapreduce.am.staging-dir</name>
+ <value>/user</value>
+ </property>
+
+ <property>
+ <name>mapred.child.java.opts</name>
+ <value>-Xmx1024m</value>
</property>
<% if has_variable?("hadoop_config_mapred_child_ulimit") %>
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/taskcontroller.cfg
URL:
http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/taskcontroller.cfg?rev=1208881&r1=1208880&r2=1208881&view=diff
==============================================================================
---
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/taskcontroller.cfg
(original)
+++
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/taskcontroller.cfg
Wed Nov 30 23:31:47 2011
@@ -1,4 +1,4 @@
-mapred.local.dir=<% hadoop_storage_locations.split(";").each do
|storage_location| %><%= storage_location%>/scratch,<% end %>
+mapred.local.dir=<% hadoop_storage_locations.split(";").each do
|storage_location| %><%= storage_location%>/mapred,<% end %>
hadoop.log.dir=/var/log/hadoop/
mapreduce.tasktracker.group=mapred
min.user.id=0
Added:
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml
URL:
http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml?rev=1208881&view=auto
==============================================================================
---
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml
(added)
+++
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml
Wed Nov 30 23:31:47 2011
@@ -0,0 +1,52 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+ <property>
+ <name>yarn.resourcemanager.resource-tracker.address</name>
+ <value><%= hadoop_rm_host %>:<%= hadoop_rt_port %></value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value><%= hadoop_rm_host %>:<%= hadoop_rm_port %></value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.scheduler.address</name>
+ <value><%= hadoop_rm_host %>:<%= hadoop_sc_port %></value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.aux-services</name>
+ <value>mapreduce.shuffle</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
+ <value>org.apache.hadoop.mapred.ShuffleHandler</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.local-dirs</name>
+ <value><% hadoop_storage_locations.split(";").each do |storage_location|
%><%= storage_location%>/yarn,<% end %></value>
+ <final>true</final>
+ </property>
+
+</configuration>
