Author: rvs
Date: Fri Oct 28 22:56:03 2011
New Revision: 1190652
URL: http://svn.apache.org/viewvc?rev=1190652&view=rev
Log:
BIGTOP-206. during deployment time hdfs/mapred dir creation should be managed
by hadoop module
Added:
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/.init.pp.swp
Modified:
incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp
URL:
http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp?rev=1190652&r1=1190651&r2=1190652&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp Fri Oct 28
22:56:03 2011
@@ -33,7 +33,10 @@ class hadoop_cluster_node {
$hadoop_zookeeper_ensemble = ["$hadoop_head_node:2888:3888"]
-
+ $namenode_data_dirs = ["/mnt/namenode"]
+ $hdfs_data_dirs = ["/mnt/hdfs"]
+ $mapred_data_dirs = ["/mnt/scratch"]
+
if ($hadoop_security_authentication == "kerberos") {
$kerberos_domain = "compute-1.internal"
$kerberos_realm = "EXAMPLE.COM"
@@ -53,6 +56,7 @@ class hadoop_worker_node inherits hadoop
hadoop::datanode { "datanode":
namenode_host => $hadoop_namenode_host,
namenode_port => $hadoop_namenode_port,
+ dirs => $hdfs_data_dirs,
auth => $hadoop_security_authentication,
}
@@ -61,6 +65,7 @@ class hadoop_worker_node inherits hadoop
namenode_port => $hadoop_namenode_port,
jobtracker_host => $hadoop_jobtracker_host,
jobtracker_port => $hadoop_jobtracker_port,
+ dirs => $mapred_data_dirs,
auth => $hadoop_security_authentication,
}
@@ -69,25 +74,6 @@ class hadoop_worker_node inherits hadoop
zookeeper_quorum => $hadoop_hbase_zookeeper_quorum,
kerberos_realm => $kerberos_realm,
}
-
- $hdfs_data_dir = ["/mnt/hdfs"]
- $mapred_data_dir = [ "/mnt/scratch" ]
-
- file {
- $mapred_data_dir:
- ensure => directory,
- owner => mapred,
- group => mapred,
- mode => 755,
- }
-
- file {
- $hdfs_data_dir:
- ensure => directory,
- owner => hdfs,
- group => hdfs,
- mode => 755,
- }
}
class hadoop_head_node inherits hadoop_cluster_node {
@@ -100,6 +86,7 @@ class hadoop_head_node inherits hadoop_c
port => $hadoop_namenode_port,
jobtracker_host => $hadoop_jobtracker_host,
jobtracker_port => $hadoop_jobtracker_port,
+ dirs => $namenode_data_dirs,
# thrift_port => $hadoop_namenode_thrift_port,
auth => $hadoop_security_authentication,
}
@@ -115,6 +102,7 @@ class hadoop_head_node inherits hadoop_c
namenode_port => $hadoop_namenode_port,
host => $hadoop_jobtracker_host,
port => $hadoop_jobtracker_port,
+ dirs => $mapred_data_dirs,
# thrift_port => $hadoop_jobtracker_thrift_port,
auth => $hadoop_security_authentication,
}
@@ -134,24 +122,6 @@ class hadoop_head_node inherits hadoop_c
ensemble => $hadoop_zookeeper_ensemble,
}
- $namenode_data_dir = ["/mnt/namenode"]
-
- file { $namenode_data_dir:
- ensure => directory,
- owner => hdfs,
- group => hdfs,
- mode => 700,
- }
-
- $mapred_data_dir = ["/mnt/scratch"]
-
- file { $mapred_data_dir:
- ensure => directory,
- owner => mapred,
- group => mapred,
- mode => 755,
- }
-
hadoop::create_hdfs_dirs { [ "/mapred", "/tmp", "/system", "/user",
"/hbase", "/benchmarks", "/user/jenkins", "/user/hive" ]:
hdfs_dirs_meta => { "/tmp" => { perm => "777", user => "hdfs" },
"/mapred" => { perm => "755", user => "mapred" },
Added:
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/.init.pp.swp
URL:
http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/.init.pp.swp?rev=1190652&view=auto
==============================================================================
Files
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/.init.pp.swp
(added) and
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/.init.pp.swp
Fri Oct 28 22:56:03 2011 differ
Modified:
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
URL:
http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp?rev=1190652&r1=1190651&r2=1190652&view=diff
==============================================================================
---
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
(original)
+++
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
Fri Oct 28 22:56:03 2011
@@ -56,7 +56,7 @@ class hadoop {
}
- define datanode ($namenode_host, $namenode_port, $port = "50075", $auth =
"simple") {
+ define datanode ($namenode_host, $namenode_port, $port = "50075", $auth =
"simple", $dirs = ["/tmp/data"]) {
$hadoop_namenode_host = $namenode_host
$hadoop_namenode_port = $namenode_port
@@ -70,19 +70,25 @@ class hadoop {
require => Package["jdk"],
}
- service { "hadoop-datanode":
- ensure => running,
- hasstatus => true,
- subscribe => [Package["hadoop-datanode"],
File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"],
File["/etc/hadoop/conf/hadoop-env.sh"]],
- require => [ Package["hadoop-datanode"] ],
- }
-
if ($hadoop_security_authentication == "kerberos") {
package { "hadoop-sbin":
ensure => latest,
require => [Package["hadoop"], Yumrepo["Bigtop"]],
}
}
+
+ service { "hadoop-datanode":
+ ensure => running,
+ hasstatus => true,
+ subscribe => [Package["hadoop-datanode"],
File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"],
File["/etc/hadoop/conf/hadoop-env.sh"]],
+ require => [ Package["hadoop-datanode"] ],
+ } -> file { $dirs:
+ ensure => directory,
+ owner => hdfs,
+ group => hdfs,
+ mode => 755,
+ require => [Package["hadoop"]],
+ }
}
define create_hdfs_dirs($hdfs_dirs_meta) {
@@ -97,7 +103,7 @@ class hadoop {
}
}
- define namenode ($jobtracker_host, $jobtracker_port, $host = $fqdn , $port =
"8020", $thrift_port= "10090", $auth = "simple") {
+ define namenode ($jobtracker_host, $jobtracker_port, $host = $fqdn , $port =
"8020", $thrift_port= "10090", $auth = "simple", $dirs = ["/tmp/nn"]) {
$hadoop_namenode_host = $host
$hadoop_namenode_port = $port
@@ -118,6 +124,11 @@ class hadoop {
hasstatus => true,
subscribe => [Package["hadoop-namenode"],
File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
require => [Package["hadoop-namenode"], Exec["namenode format"]],
+ } -> file { $dirs:
+ ensure => directory,
+ owner => hdfs,
+ group => hdfs,
+ mode => 700,
}
exec { "namenode format":
@@ -129,7 +140,7 @@ class hadoop {
}
- define jobtracker ($namenode_host, $namenode_port, $host = $fqdn, $port =
"8021", $thrift_port = "9290", $auth = "simple") {
+ define jobtracker ($namenode_host, $namenode_port, $host = $fqdn, $port =
"8021", $thrift_port = "9290", $auth = "simple", $dirs = ["/tmp/mr"]) {
$hadoop_namenode_host = $namenode_host
$hadoop_namenode_port = $namenode_port
@@ -150,11 +161,17 @@ class hadoop {
hasstatus => true,
subscribe => [Package["hadoop-jobtracker"],
File["/etc/hadoop/conf/core-site.xml"],
File["/etc/hadoop/conf/mapred-site.xml"],
File["/etc/hadoop/conf/hadoop-env.sh"]],
require => [ Package["hadoop-jobtracker"] ]
+ } -> file { $dirs:
+ ensure => directory,
+ owner => mapred,
+ group => mapred,
+ mode => 755,
+ require => [Package["hadoop"]],
}
}
- define tasktracker ($namenode_host, $namenode_port, $jobtracker_host,
$jobtracker_port, $auth = "simple"){
+ define tasktracker ($namenode_host, $namenode_port, $jobtracker_host,
$jobtracker_port, $auth = "simple", $dirs = ["/tmp/mr"]){
$hadoop_namenode_host = $namenode_host
$hadoop_namenode_port = $namenode_port
@@ -178,6 +195,12 @@ class hadoop {
hasstatus => true,
subscribe => [Package["hadoop-tasktracker"],
File["/etc/hadoop/conf/core-site.xml"],
File["/etc/hadoop/conf/mapred-site.xml"],
File["/etc/hadoop/conf/hadoop-env.sh"]],
require => [ Package["hadoop-tasktracker"],
File["/etc/hadoop/conf/taskcontroller.cfg"] ],
+ } -> file { $dirs:
+ ensure => directory,
+ owner => mapred,
+ group => mapred,
+ mode => 755,
+ require => [Package["hadoop"]],
}
}
