Repository: bigtop Updated Branches: refs/heads/master 3b4ae8aba -> 7d76e1c12
BIGTOP-1417. Dockerize the puppetized vagrant deployer Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/7d76e1c1 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/7d76e1c1 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/7d76e1c1 Branch: refs/heads/master Commit: 7d76e1c12d96760944e495c7777acd4e8efb4142 Parents: 3b4ae8a Author: evans_ye <[email protected]> Authored: Tue Oct 21 03:34:26 2014 +0800 Committer: [email protected] <jayunit100> Committed: Mon Oct 20 22:18:56 2014 -0400 ---------------------------------------------------------------------- bigtop-deploy/vm/docker-puppet/Dockerfile | 33 +++++ bigtop-deploy/vm/docker-puppet/README.md | 123 +++++++++++++++++++ bigtop-deploy/vm/docker-puppet/Vagrantfile | 69 +++++++++++ .../vm/docker-puppet/boot2docker/Vagrantfile | 18 +++ bigtop-deploy/vm/docker-puppet/docker-hadoop.sh | 88 +++++++++++++ bigtop-deploy/vm/docker-puppet/provision.sh | 45 +++++++ 6 files changed, 376 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/7d76e1c1/bigtop-deploy/vm/docker-puppet/Dockerfile ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/docker-puppet/Dockerfile b/bigtop-deploy/vm/docker-puppet/Dockerfile new file mode 100644 index 0000000..8e65126 --- /dev/null +++ b/bigtop-deploy/vm/docker-puppet/Dockerfile @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM bigtop/seed:centos-6.4 + +# enable ssh +RUN yum -y install openssh-server openssh-clients sudo +RUN sed -i.bak s/UsePAM\ yes/UsePAM\ no/ /etc/ssh/sshd_config +RUN ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key +RUN ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key + +# requiretty off +RUN sed -i.bak 's/requiretty/!requiretty/' /etc/sudoers + +# setup vagrant account +RUN mkdir /root/.ssh/ +RUN chmod 0755 /root/.ssh +RUN wget http://github.com/mitchellh/vagrant/raw/master/keys/vagrant.pub --no-check-certificate -O /root/.ssh/authorized_keys +RUN chmod 0644 /root/.ssh/authorized_keys + +CMD /usr/sbin/sshd -D http://git-wip-us.apache.org/repos/asf/bigtop/blob/7d76e1c1/bigtop-deploy/vm/docker-puppet/README.md ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/docker-puppet/README.md b/bigtop-deploy/vm/docker-puppet/README.md new file mode 100644 index 0000000..3b54bea --- /dev/null +++ b/bigtop-deploy/vm/docker-puppet/README.md @@ -0,0 +1,123 @@ + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +------------------------------------------------------------------------------------------------------------------------------------------------------ + +#BigTop docker provisioner + +## Overview + +The Vagrantfile definition and wrapper script that creates Bigtop virtual Hadoop cluster on top of Docker containers for you, by pulling from existing publishing bigtop repositories. +This cluster can be used: + +- to test bigtop smoke tests +- to test bigtop puppet recipes + +## Prerequisites + +### OS X and Windows + +* Install [VirtualBox](https://www.virtualbox.org/wiki/Downloads) + +* Install [Vagrant](http://www.vagrantup.com/downloads.html) + +### Linux + +* [Kernel Requirements](http://docker.readthedocs.org/en/v0.5.3/installation/kernel/) + +* Install [Docker](https://docs.docker.com/installation/) + +* Install [Vagrant](http://www.vagrantup.com/downloads.html) + +## Getting Started + +* Create a 3 node Bigtop Hadoop cluster from scratch + +``` +cd bigtop/bigtop-deploy/vm/docker-puppet +./docker-hadoop.sh --build-image --create 3 +``` + +## USAGE + +1) Build up the base Docker image that supports Vagrant. + +``` +./docker-hadoop.sh --build-image +``` + +2) Create a Bigtop Hadoop cluster by given # of node. (will place a file called config.rb) + +``` +./docker-hadoop.sh --create 3 +``` + +3) Destroy the cluster. + +``` +./docker-hadoop.sh --destroy +``` + +4) Update your cluster after doing configuration changes. (re-run puppet apply) + +``` +./docker-hadoop.sh --provision +``` + +5) Chain your operations with-in one command. + +``` +./docker-hadoop.sh --build-image --create 5 --destroy +``` + +Commands will be executed by following order: + +``` +build-image => create 5 node cluster => destroy the cluster +``` + +6) Run hbase-test.sh to evaluate the deployment. + +``` +../vagrant-puppet/hbase-test.sh +``` + +7) See helper message: + +``` +./docker-hadoop.sh -h +usage: docker-hadoop.sh [options] + -b, --build-image Build base Docker image for Bigtop Hadoop + (must be exectued at least once before creating cluster) + -c NUM_INSTANCES, --create NUM_INSTANCES Create a docker based Bigtop Hadoop cluster + -p, --provision Deploy configuration changes + -d, --destroy Destroy the cluster + -h, --help +``` + +##Configure Apache Hadoop ecosystem components + +* Choose the ecosystem you want to be deployed by modifying components in provision.sh. + +``` + components,hadoop,hbase,yarn,mapred-app,... +``` + +By default, Apache Hadoop, YARN, and Apache HBase will be installed. +See `bigtop-deploy/puppet/config/site.csv.example` for more details. + +##Notes + +* Users currently using vagrant 1.6+ is strongly recommanded to upgrade to 1.6.4+, otherwise you will encounter the [issue](https://github.com/mitchellh/vagrant/issues/3769) when installing plguins http://git-wip-us.apache.org/repos/asf/bigtop/blob/7d76e1c1/bigtop-deploy/vm/docker-puppet/Vagrantfile ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/docker-puppet/Vagrantfile b/bigtop-deploy/vm/docker-puppet/Vagrantfile new file mode 100755 index 0000000..ea7153a --- /dev/null +++ b/bigtop-deploy/vm/docker-puppet/Vagrantfile @@ -0,0 +1,69 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'fileutils' + +# number of instances +$num_instances = 3 +# the bigtop puppet recipes +$bigtop_puppet = "../../puppet" + +# load configuration from config.rb +CONFIG = File.join(File.dirname(__FILE__), "config.rb") +if File.exist?(CONFIG) + require CONFIG +end + +ENV['VAGRANT_DEFAULT_PROVIDER'] ||= 'docker' + +VAGRANTFILE_API_VERSION = "2" +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + + # image definition + config.vm.define "image", autostart: false do |image| + image.vm.provider "docker" do |d| + d.build_dir = "." + d.build_args = ["-t","bigtop/ssh:centos-6.4"] + d.vagrant_machine = "boot2docker" + d.vagrant_vagrantfile = "./boot2docker/Vagrantfile" + end + image.ssh.username = "root" + image.ssh.port = 22 + end + + # nodes definition + (1..$num_instances).each do |i| + config.vm.define "bigtop#{i}" do |bigtop| + # docker container settings + bigtop.vm.provider "docker" do |d| + d.image = "bigtop/ssh:centos-6.4" + d.create_args = ["--privileged=true"] + if "#{i}" == "1" + d.ports = ["50070:50070","8088:8088","60010:60010"] + end + d.vagrant_machine = "boot2docker" + d.vagrant_vagrantfile = "./boot2docker/Vagrantfile" + end + bigtop.ssh.username = "root" + bigtop.ssh.port = 22 + bigtop.vm.hostname = "bigtop#{i}.docker" + bigtop.vm.synced_folder $bigtop_puppet, "/bigtop-puppet" + end + end + +end http://git-wip-us.apache.org/repos/asf/bigtop/blob/7d76e1c1/bigtop-deploy/vm/docker-puppet/boot2docker/Vagrantfile ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/docker-puppet/boot2docker/Vagrantfile b/bigtop-deploy/vm/docker-puppet/boot2docker/Vagrantfile new file mode 100644 index 0000000..8fa04f8 --- /dev/null +++ b/bigtop-deploy/vm/docker-puppet/boot2docker/Vagrantfile @@ -0,0 +1,18 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +VAGRANTFILE_API_VERSION = "2" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + + config.vm.define "boot2docker" do |node| + node.vm.box = "yungsang/boot2docker" + node.vm.provider "virtualbox" do |vb| + vb.customize ["modifyvm", :id, "--memory", "4196"] + end + node.vm.network "forwarded_port", host: 50070, guest: 50070 + node.vm.network "forwarded_port", host: 8088, guest: 8088 + node.vm.network "forwarded_port", host: 60010, guest: 60010 + end + +end http://git-wip-us.apache.org/repos/asf/bigtop/blob/7d76e1c1/bigtop-deploy/vm/docker-puppet/docker-hadoop.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/docker-puppet/docker-hadoop.sh b/bigtop-deploy/vm/docker-puppet/docker-hadoop.sh new file mode 100755 index 0000000..449b5fb --- /dev/null +++ b/bigtop-deploy/vm/docker-puppet/docker-hadoop.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +usage() { + echo "usage: $PROG [options]" + echo " -b, --build-image Build base Docker image for Bigtop Hadoop" + echo " (must be exectued at least once before creating cluster)" + echo " -c NUM_INSTANCES, --create=NUM_INSTANCES Create a docker based Bigtop Hadoop cluster" + echo " -p, --provision Deploy configuration changes" + echo " -d, --destroy Destroy the cluster" + echo " -h, --help" + exit 1 +} + +build-image() { + vagrant up image --provider docker + { + echo "echo -e '\nBUILD IMAGE SUCCESS.\n'" |vagrant ssh image + } || { + >&2 echo -e "\nBUILD IMAGE FAILED!\n" + exit 2 + } + vagrant destroy image -f +} + +create() { + echo "\$num_instances = $1" > config.rb + vagrant up --no-parallel + nodes=(`vagrant status |grep running |awk '{print $1}'`) + hadoop_head_node=(`echo "hostname -f" |vagrant ssh ${nodes[0]} |tail -n 1`) + echo "/vagrant/provision.sh $hadoop_head_node" |vagrant ssh ${nodes[0]} + bigtop-puppet ${nodes[0]} + for ((i=1 ; i<${#nodes[*]} ; i++)); do + ( + echo "/vagrant/provision.sh $hadoop_head_node" |vagrant ssh ${nodes[$i]} + bigtop-puppet ${nodes[$i]} + ) & + done + wait +} + +provision() { + nodes=(`vagrant status |grep running |awk '{print $1}'`) + for node in $nodes; do + bigtop-puppet $node & + done + wait +} + +destroy() { + vagrant destroy -f + rm -f ./hosts +} + +bigtop-puppet() { + echo "puppet apply -d --confdir=/bigtop-puppet --modulepath=/bigtop-puppet/modules /bigtop-puppet/manifests/site.pp" |vagrant ssh $1 +} + +PROG=`basename $0` +ARGS=`getopt -o "bc:pdh" -l "build-image,create:,provision,destroy,help" -n $PROG -- "$@"` + +if [ $? -ne 0 ]; then + usage +fi + +eval set -- "$ARGS" + +while true; do + case "$1" in + -b|--build-image) + build-image + shift;; + -c|--create) + create $2 + shift 2;; + -p|--provision) + provision + shift;; + -d|--destroy) + destroy + shift;; + -h|--help) + usage + shift;; + --) + shift + break;; + esac +done http://git-wip-us.apache.org/repos/asf/bigtop/blob/7d76e1c1/bigtop-deploy/vm/docker-puppet/provision.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/docker-puppet/provision.sh b/bigtop-deploy/vm/docker-puppet/provision.sh new file mode 100755 index 0000000..929999d --- /dev/null +++ b/bigtop-deploy/vm/docker-puppet/provision.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Install puppet agent +yum -y install http://yum.puppetlabs.com/puppetlabs-release-el-6.noarch.rpm +yum -y install puppet-2.7.24-1.el6.noarch + +sysctl kernel.hostname=`hostname -f` + +# Unmount device /etc/hosts and replace it by a shared hosts file +echo -e "`hostname -i`\t`hostname -f`" >> /vagrant/hosts +umount /etc/hosts +mv /etc/hosts /etc/hosts.bak +ln -s /vagrant/hosts /etc/hosts + +# Prepare puppet configuration file +cat > /bigtop-puppet/config/site.csv << EOF +hadoop_head_node,$1 +hadoop_storage_dirs,/data/1,/data/2 +bigtop_yumrepo_uri,http://bigtop01.cloudera.org:8080/view/Releases/job/Bigtop-0.8.0/label=centos6/6/artifact/output/ +jdk_package_name,java-1.7.0-openjdk-devel.x86_64 +components,hadoop,hbase,yarn,mapred-app +EOF + +mkdir -p /data/{1,2} + +# Setup rng-tools to improve virtual machine entropy performance. +# The poor entropy performance will cause kerberos provisioning failed. +yum -y install rng-tools +sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd +service rngd start
