Ottomata has submitted this change and it was merged.
Change subject: Updates for CDH 5.3.1 (Hadoop)
......................................................................
Updates for CDH 5.3.1 (Hadoop)
Bug: T70302
Change-Id: I80083f8c07f372cb4f8a8c6050fc3af4b3bb9c4f
---
M puppet/modules/cdh
M puppet/modules/role/manifests/analytics.pp
M puppet/modules/role/manifests/hadoop.pp
M puppet/modules/role/manifests/hive.pp
4 files changed, 32 insertions(+), 23 deletions(-)
Approvals:
Ottomata: Looks good to me, approved
diff --git a/puppet/modules/cdh b/puppet/modules/cdh
index 65335f7..6e5d310 160000
--- a/puppet/modules/cdh
+++ b/puppet/modules/cdh
-Subproject commit 65335f7d027c635065966a15a7b4467c1ce311b5
+Subproject commit 6e5d3103894cd1a64ffa323a28fdc40611a576a1
diff --git a/puppet/modules/role/manifests/analytics.pp
b/puppet/modules/role/manifests/analytics.pp
index 297faf8..e8ef66d 100644
--- a/puppet/modules/role/manifests/analytics.pp
+++ b/puppet/modules/role/manifests/analytics.pp
@@ -25,36 +25,20 @@
# 11000: 11000 # Oozie
#
# ALSO! If you are not planning on using Mediawiki for this
-# vagrant instance, you should comment out the include mediawiki
-# line in site.pp. You'll end up installing much less stuff.
+# vagrant instance, you should edit hieradata/common.yaml and
+# remove the - '::role::mediawiki' line from classes to include
+# by default.
#
class role::analytics {
include ::apt
- # CDH packages are not yet packaged for Trusty.
- # In the meantime, we allow Vagrant Trusty to
- # install Precise packages.
- file { '/etc/apt/sources.list.d/wikimedia-precise.list':
- source => 'puppet:///files/apt/wikimedia-precise.list',
- notify => Exec['update_package_index'],
- before => Class['::role::hadoop'],
- }
-
include ::role::hadoop
include ::role::hive
include ::role::oozie
- # cdh::pig and cdh::sqoop are not parameterized so they
+ include ::role::hue
+ # cdh::mahout, cdh::pig and cdh::sqoop are not parameterized so they
# do not need their own role classes.
+ include ::cdh::mahout
include ::cdh::pig
include ::cdh::sqoop
-
- # Hue does not currently work with Trusty :(
- # Need to solve this error:
- # File
"/usr/lib/hue/build/env/lib/python2.7/site-packages/python_daemon-1.5.1-py2.7.egg/daemon/daemon.py",
line 25, in <module>
- # import resource
- # ImportError: No module named resource
- # I think this has something to do with the fact that
- # this symlink is broken in Trusty:
- # /usr/lib/hue/build/env/lib/python2.7/config -> /usr/lib/python2.7/config
- #include ::role::hue
}
diff --git a/puppet/modules/role/manifests/hadoop.pp
b/puppet/modules/role/manifests/hadoop.pp
index 249bf85..da45cef 100644
--- a/puppet/modules/role/manifests/hadoop.pp
+++ b/puppet/modules/role/manifests/hadoop.pp
@@ -76,4 +76,29 @@
subscribe => [Service['hadoop-hdfs-namenode'],
Service['hadoop-hdfs-datanode']],
refreshonly => true,
}
+
+ # This packages conflicts with the hadoop-fuse-dfs
+ # script in that two libjvm.so files get added
+ # to LD_LIBRARY_PATH. We dont't need this
+ # package anyway, so ensure it is absent.
+ package { 'icedtea-7-jre-jamvm':
+ ensure => 'absent'
+ }
+ # Mount HDFS via Fuse on Analytics client nodes.
+ # This will mount HDFS at /mnt/hdfs read only.
+ class { 'cdh::hadoop::mount':
+ # Make sure this package is removed before
+ # cdh::hadoop::mount evaluates.
+ require => [
+ Package['icedtea-7-jre-jamvm'],
+ Exec['wait_for_hdfs'],
+ ]
+ }
+
+ # Create HDFS /user/vagrant hdfs homedir.
+ cdh::hadoop::directory { '/user/vagrant':
+ owner => 'vagrant',
+ group => 'vagrant',
+ require => Exec['wait_for_hdfs'],
+ }
}
diff --git a/puppet/modules/role/manifests/hive.pp
b/puppet/modules/role/manifests/hive.pp
index 2266de8..0c2c401 100644
--- a/puppet/modules/role/manifests/hive.pp
+++ b/puppet/modules/role/manifests/hive.pp
@@ -11,7 +11,7 @@
metastore_host => $role::hadoop::namenode_hosts[0],
db_root_password => $::mysql::root_password,
# $hive_version and $cdh_version are custom facts added by the cdh
module.
- auxpath =>
'file:///usr/lib/hive-hcatalog/share/hcatalog/hive-hcatalog-core-0.12.0-cdh5.0.2.jar',
+ auxpath =>
'file:///usr/lib/hive-hcatalog/share/hcatalog/hive-hcatalog-core.jar'
}
# Setup Hive server and Metastore
--
To view, visit https://gerrit.wikimedia.org/r/191472
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I80083f8c07f372cb4f8a8c6050fc3af4b3bb9c4f
Gerrit-PatchSet: 2
Gerrit-Project: mediawiki/vagrant
Gerrit-Branch: master
Gerrit-Owner: Ottomata <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
