Repository: ambari Updated Branches: refs/heads/trunk 148f5a9b8 -> b42c3c9be
AMBARI-7875 Docker build on builds.a.o and support JDK1.6 and python2.6 (jaoki) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b42c3c9b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b42c3c9b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b42c3c9b Branch: refs/heads/trunk Commit: b42c3c9bed122368d66e317d8943071a69534ac2 Parents: 148f5a9 Author: Jun Aoki <ja...@apache.org> Authored: Mon Oct 27 13:02:54 2014 -0700 Committer: Jun Aoki <ja...@apache.org> Committed: Mon Oct 27 13:02:54 2014 -0700 ---------------------------------------------------------------------- dev-support/docker/README.md | 51 ++++ dev-support/docker/docker/Dockerfile | 79 +++++++ dev-support/docker/docker/bin/__init__.py | 12 + dev-support/docker/docker/bin/ambaribuild.py | 233 +++++++++++++++++++ dev-support/docker/docker/bin/test/__init__.py | 12 + .../docker/docker/bin/test/ambaribuild_test.py | 70 ++++++ .../single-node-HDP-2.1-blueprint1.json | 64 +++++ .../blueprints/single-node-hostmapping1.json | 14 ++ 8 files changed, 535 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b42c3c9b/dev-support/docker/README.md ---------------------------------------------------------------------- diff --git a/dev-support/docker/README.md b/dev-support/docker/README.md new file mode 100644 index 0000000..6613e0d --- /dev/null +++ b/dev-support/docker/README.md @@ -0,0 +1,51 @@ +<!-- +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> + + +how to build +-------------------- + +``` +docker build -t ambari/build ./docker +``` + +how to run +-------------------- + +``` +# bash +docker run --privileged -t -i -p 80:80 -p 5005:5005 -p 8080:8080 -h node1.mydomain.com --name ambari1 -v ${AMBARI_SRC:-$(pwd)}:/tmp/ambari ambari/build bash +# where 5005 is java debug port and 8080 is the default http port, if no --privileged ambari-server start fails due to access to /proc/??/exe +# -t is required otherwise, sudo commands do not run + +# build, install ambari and deploy hadoop in container +cd {ambari src} +docker rm ambari1 +docker run --privileged -t -p 80:80 -p 5005:5005 -p 8080:8080 -h node1.mydomain.com --name ambari1 -v ${AMBARI_SRC:-$(pwd)}:/tmp/ambari ambari/build /tmp/ambari-build-docker/bin/ambaribuild.py [test|server|agent|deploy] [-b] [-s [HDP|BIGTOP|PHD]] +where +test: mvn test +server: install and run ambari-server +agent: install and run ambari-server and ambari-agent +deploy: install and run ambari-server and ambari-agent, and deploy a hadoop +-b option to rebuild ambari +``` + +how to run unit test +-------------------- +``` +cd docker +python -m bin.test.ambaribuild_test + +``` + http://git-wip-us.apache.org/repos/asf/ambari/blob/b42c3c9b/dev-support/docker/docker/Dockerfile ---------------------------------------------------------------------- diff --git a/dev-support/docker/docker/Dockerfile b/dev-support/docker/docker/Dockerfile new file mode 100644 index 0000000..96ba8ff --- /dev/null +++ b/dev-support/docker/docker/Dockerfile @@ -0,0 +1,79 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM centos:centos6 + +RUN echo root:changeme | chpasswd + +## Install some basic utilities that aren't in the default image +RUN yum -y install vim wget rpm-build sudo which telnet tar openssh-server openssh-clients ntp git python-setuptools httpd +# phantomjs dependency +RUN yum -y install fontconfig freetype libfreetype.so.6 libfontconfig.so.1 libstdc++.so.6 +RUN rpm -e --nodeps --justdb glibc-common +RUN yum -y install glibc-common + +ENV HOME /root + +#Install JAVA +# RUN wget --no-check-certificate --no-cookies --header "Cookie:oraclelicense=accept-securebackup-cookie" http://download.oracle.com/otn-pub/java/jdk/7u55-b13/jdk-7u55-linux-x64.rpm -O jdk-7u55-linux-x64.rpm +# RUN yum -y install jdk-7u55-linux-x64.rpm +RUN wget --no-check-certificate --no-cookies --header "Cookie:oraclelicense=accept-securebackup-cookie" http://download.oracle.com/otn-pub/java/jdk/6u45-b06/jdk-6u45-linux-x64-rpm.bin -O jdk-6u45-linux-x64-rpm.bin +RUN chmod +x jdk-6u45-linux-x64-rpm.bin +RUN ./jdk-6u45-linux-x64-rpm.bin +ENV JAVA_HOME /usr/java/default/ + +#Install Maven +RUN mkdir -p /opt/maven +WORKDIR /opt/maven +RUN wget http://apache.cs.utah.edu/maven/maven-3/3.0.5/binaries/apache-maven-3.0.5-bin.tar.gz +RUN tar -xvzf /opt/maven/apache-maven-3.0.5-bin.tar.gz +RUN rm -rf /opt/maven/apache-maven-3.0.5-bin.tar.gz + +ENV M2_HOME /opt/maven/apache-maven-3.0.5 +ENV MAVEN_OPTS -Xmx2048m -XX:MaxPermSize=256m +ENV PATH $PATH:$JAVA_HOME/bin:$M2_HOME/bin + + +# SSH key +RUN ssh-keygen -f /root/.ssh/id_rsa -t rsa -N '' +RUN cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys +RUN chmod 600 /root/.ssh/authorized_keys +RUN sed -ri 's/UsePAM yes/UsePAM no/g' /etc/ssh/sshd_config + +# Install python, nodejs and npm +RUN yum -y install http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm +RUN yum -y install nodejs npm --enablerepo=epel +RUN npm install -g brunch@1.7.13 + +# Once run some mvn commands to cache .m2/repository +WORKDIR /tmp +RUN git clone https://github.com/apache/ambari.git +WORKDIR /tmp/ambari +RUN mvn versions:set -DnewVersion=1.6.1.0 +RUN mvn -B clean install package rpm:rpm -DskipTests -DnewVersion=1.6.1.0 -Dpython.ver="python >= 2.6" -Preplaceurl + +# also build ambari-log4j and install +WORKDIR /tmp/ambari/contrib/ambari-log4j +RUN mvn package rpm:rpm +RUN yum install -y target/rpm/ambari-log4j/RPMS/noarch/ambari-log4j-1.2.1-*.noarch.rpm + +# clean git code because I want to use the one on local filesystem. +WORKDIR /tmp +RUN rm -rf /tmp/ambari + +RUN mkdir -p /tmp/ambari-build-docker/blueprints +ADD ./blueprints /tmp/ambari-build-docker/blueprints +RUN mkdir -p /tmp/ambari-build-docker/bin +ADD ./bin /tmp/ambari-build-docker/bin + +WORKDIR /tmp + http://git-wip-us.apache.org/repos/asf/ambari/blob/b42c3c9b/dev-support/docker/docker/bin/__init__.py ---------------------------------------------------------------------- diff --git a/dev-support/docker/docker/bin/__init__.py b/dev-support/docker/docker/bin/__init__.py new file mode 100644 index 0000000..fac912e --- /dev/null +++ b/dev-support/docker/docker/bin/__init__.py @@ -0,0 +1,12 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + http://git-wip-us.apache.org/repos/asf/ambari/blob/b42c3c9b/dev-support/docker/docker/bin/ambaribuild.py ---------------------------------------------------------------------- diff --git a/dev-support/docker/docker/bin/ambaribuild.py b/dev-support/docker/docker/bin/ambaribuild.py new file mode 100755 index 0000000..66df31f --- /dev/null +++ b/dev-support/docker/docker/bin/ambaribuild.py @@ -0,0 +1,233 @@ +#!/usr/bin/python +# coding: utf-8 +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess, time, sys +import json +from optparse import OptionParser + +SKIP_TEST="-DskipTests" +AMBARI_AUTH_HEADERS = "--header 'Authorization:Basic YWRtaW46YWRtaW4=' --header 'X-Requested-By: PIVOTAL'" +AMBARI_BUILD_DOCKER_ROOT = "/tmp/ambari-build-docker" +NO_EXIT_SLEEP_TIME=60 +RETRY_MAX=20 + +def git_deep_cleaning(): + proc = subprocess.Popen("git clean -xdf", + shell=True, + cwd="/tmp/ambari") + return proc.wait() + +def ambariUnitTest(): + git_deep_cleaning() + proc = subprocess.Popen("mvn -fae clean install", + shell=True, + cwd="/tmp/ambari") + return proc.wait() + +def buildAmbari(stack_distribution): + git_deep_cleaning() + stack_distribution_param = "" + if stack_distribution is not None: + stack_distribution_param = "-Dstack.distribution=" + stack_distribution + proc = subprocess.Popen("mvn -B clean install package rpm:rpm -Dmaven.clover.skip=true -Dfindbugs.skip=true " + + SKIP_TEST + " " + + stack_distribution_param + " -Dpython.ver=\"python >= 2.6\"", + shell=True, + cwd="/tmp/ambari") + return proc.wait() + +def install_ambari_server(): + proc = subprocess.Popen("sudo yum install -y ambari-server-*.noarch.rpm", + shell=True, + cwd="/tmp/ambari/ambari-server/target/rpm/ambari-server/RPMS/noarch") + return proc.wait() + +def install_ambari_agent(): + proc = subprocess.Popen("sudo yum install -y ambari-agent-*.x86_64.rpm", + shell=True, + cwd="/tmp/ambari/ambari-agent/target/rpm/ambari-agent/RPMS/x86_64") + return proc.wait() + +def setup_ambari_server(): + proc = subprocess.Popen("echo -e '\n\n\n\n' | sudo ambari-server setup", + shell=True) + return proc.wait() + +def start_ambari_server(debug=False): + proc = subprocess.Popen("sudo ambari-server start" + (" --debug" if debug else ""), + shell=True) + return proc.wait() + +def start_dependant_services(): + retcode = 0 + proc = subprocess.Popen("sudo service sshd start", shell=True) + retcode += proc.wait() + proc = subprocess.Popen("sudo service ntpd start", shell=True) + retcode += proc.wait() + return retcode + +def configure_ambari_agent(): + proc = subprocess.Popen("hostname -f", stdout=subprocess.PIPE, shell=True) + hostname = proc.stdout.read().rstrip() + proc = subprocess.Popen("sudo sed -i 's/hostname=localhost/hostname=" + hostname + "/g' /etc/ambari-agent/conf/ambari-agent.ini", + shell=True) + return proc.wait() + +def start_ambari_agent(wait_until_registered = True): + retcode = 0 + proc = subprocess.Popen("service ambari-agent start", + shell=True) + retcode += proc.wait() + if wait_until_registered: + if not wait_until_ambari_agent_registered(): + print "ERROR: ambari-agent was not registered." + sys.exit(1) + + return retcode + +def wait_until_ambari_agent_registered(): + ''' + return True if ambari agent is found registered. + return False if timeout + ''' + count = 0 + while count < RETRY_MAX: + count += 1 + proc = subprocess.Popen("curl " + + "http://localhost:8080/api/v1/hosts " + + AMBARI_AUTH_HEADERS, + stdout=subprocess.PIPE, + shell=True) + hosts_result_string = proc.stdout.read() + hosts_result_json = json.loads(hosts_result_string) + if len(hosts_result_json["items"]) != 0: + return True + time.sleep(5) + return False + +def post_blueprint(): + proc = subprocess.Popen("curl -X POST -D - " + + "-d @single-node-HDP-2.1-blueprint1.json http://localhost:8080/api/v1/blueprints/myblueprint1 " + + AMBARI_AUTH_HEADERS , + cwd=AMBARI_BUILD_DOCKER_ROOT + "/blueprints", + shell=True) + return proc.wait() + +def create_cluster(): + proc = subprocess.Popen("curl -X POST -D - " + + "-d @single-node-hostmapping1.json http://localhost:8080/api/v1/clusters/mycluster1 " + + AMBARI_AUTH_HEADERS , + cwd=AMBARI_BUILD_DOCKER_ROOT + "/blueprints", + shell=True) + return proc.wait() + +# Loop to not to exit Docker container +def no_exit(): + print "loop to not to exit docker container..." + while True: + time.sleep(NO_EXIT_SLEEP_TIME) + +class ParseResult: + is_rebuild = False + stack_distribution = None + is_test = False + is_install_server = False + is_install_agent = False + is_deploy = False + +def parse(argv): + result = ParseResult() + if len(argv) >=2: + parser = OptionParser() + parser.add_option("-b", "--rebuild", + dest="is_rebuild", + action="store_true", + default=False, + help="set this flag if you want to rebuild Ambari code") + parser.add_option("-s", "--stack_distribution", + dest="stack_distribution", + help="set a stack distribution. [HDP|PHD|BIGTOP]. Make sure -b is also set when you set a stack distribution") + (options, args) = parser.parse_args(argv[1:]) + if options.is_rebuild: + result.is_rebuild = True + if options.stack_distribution: + result.stack_distribution = options.stack_distribution + + if argv[0] == "test": + result.is_test = True + + if argv[0] == "server": + result.is_install_server = True + + if argv[0] == "agent": + result.is_install_server = True + result.is_install_agent = True + + if argv[0] == "deploy": + result.is_install_server = True + result.is_install_agent = True + result.is_deploy = True + + return result + +if __name__ == "__main__": + + if len(sys.argv) == 1: + print "specify one of test, server, agent or deploy" + sys.exit(1) + + # test: execute unit test + # server: install ambari-server + # with or without rebuild + # agent: install ambari-server and ambari-agent + # with or without rebuild + # deploy: install ambari-server, ambari-agent and deploy Hadoop + # with or without rebuild + + parsed_args = parse(sys.argv[1:]) + + if parsed_args.is_test: + retcode = ambariUnitTest() + sys.exit(retcode) + + if parsed_args.is_rebuild: + retcode = buildAmbari(parsed_args.stack_distribution) + if retcode != 0: sys.exit(retcode) + + if parsed_args.is_install_server: + retcode = install_ambari_server() + if retcode != 0: sys.exit(retcode) + retcode = setup_ambari_server() + if retcode != 0: sys.exit(retcode) + retcode = start_ambari_server() + if retcode != 0: sys.exit(retcode) + retcode = start_dependant_services() + if retcode != 0: sys.exit(retcode) + + if parsed_args.is_install_agent: + retcode = install_ambari_agent() + if retcode != 0: sys.exit(retcode) + retcode = configure_ambari_agent() + if retcode != 0: sys.exit(retcode) + retcode = start_ambari_agent() + if retcode != 0: sys.exit(retcode) + + if parsed_args.is_deploy: + retcode = post_blueprint() + if retcode != 0: sys.exit(retcode) + retcode = create_cluster() + if retcode != 0: sys.exit(retcode) + + no_exit() + http://git-wip-us.apache.org/repos/asf/ambari/blob/b42c3c9b/dev-support/docker/docker/bin/test/__init__.py ---------------------------------------------------------------------- diff --git a/dev-support/docker/docker/bin/test/__init__.py b/dev-support/docker/docker/bin/test/__init__.py new file mode 100644 index 0000000..fac912e --- /dev/null +++ b/dev-support/docker/docker/bin/test/__init__.py @@ -0,0 +1,12 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + http://git-wip-us.apache.org/repos/asf/ambari/blob/b42c3c9b/dev-support/docker/docker/bin/test/ambaribuild_test.py ---------------------------------------------------------------------- diff --git a/dev-support/docker/docker/bin/test/ambaribuild_test.py b/dev-support/docker/docker/bin/test/ambaribuild_test.py new file mode 100755 index 0000000..223fa98 --- /dev/null +++ b/dev-support/docker/docker/bin/test/ambaribuild_test.py @@ -0,0 +1,70 @@ +#!/usr/bin/python +# coding: utf-8 +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from bin import ambaribuild + +# TODO move this to a proper location +def unittest(): + # parse + result = ambaribuild.parse(["test"]) + assert result.is_test == True + assert result.is_rebuild == False + assert result.stack_distribution == None + assert result.is_install_server == False + assert result.is_install_agent == False + assert result.is_deploy == False + + result = ambaribuild.parse(["server"]) + assert result.is_test == False + assert result.is_rebuild == False + assert result.stack_distribution == None + assert result.is_install_server == True + assert result.is_install_agent == False + assert result.is_deploy == False + + result = ambaribuild.parse(["agent"]) + assert result.is_test == False + assert result.is_rebuild == False + assert result.stack_distribution == None + assert result.is_install_server == True + assert result.is_install_agent == True + assert result.is_deploy == False + + result = ambaribuild.parse(["agent", "-b"]) + assert result.is_test == False + assert result.is_rebuild == True + assert result.stack_distribution == None + assert result.is_install_server == True + assert result.is_install_agent == True + assert result.is_deploy == False + + result = ambaribuild.parse(["deploy"]) + assert result.is_test == False + assert result.is_rebuild == False + assert result.stack_distribution == None + assert result.is_install_server == True + assert result.is_install_agent == True + assert result.is_deploy == True + + result = ambaribuild.parse(["deploy", "-b", "-s", "BIGTOP"]) + assert result.is_test == False + assert result.is_rebuild == True + assert result.stack_distribution == "BIGTOP" + assert result.is_install_server == True + assert result.is_install_agent == True + assert result.is_deploy == True + +if __name__ == "__main__": + unittest() + http://git-wip-us.apache.org/repos/asf/ambari/blob/b42c3c9b/dev-support/docker/docker/blueprints/single-node-HDP-2.1-blueprint1.json ---------------------------------------------------------------------- diff --git a/dev-support/docker/docker/blueprints/single-node-HDP-2.1-blueprint1.json b/dev-support/docker/docker/blueprints/single-node-HDP-2.1-blueprint1.json new file mode 100644 index 0000000..8ac8b2e --- /dev/null +++ b/dev-support/docker/docker/blueprints/single-node-HDP-2.1-blueprint1.json @@ -0,0 +1,64 @@ +{ + "configurations" : [ + { + "nagios-env" : { + "nagios_contact": "me@my-awesome-domain.example" + } + } + ], + "host_groups" : [ + { + "name" : "host_group_1", + "components" : [ + { + "name" : "ZOOKEEPER_SERVER" + }, + { + "name" : "ZOOKEEPER_CLIENT" + }, + { + "name" : "NAMENODE" + }, + { + "name" : "SECONDARY_NAMENODE" + }, + { + "name" : "DATANODE" + }, + { + "name" : "HDFS_CLIENT" + }, + { + "name" : "NODEMANAGER" + }, + { + "name" : "RESOURCEMANAGER" + }, + { + "name" : "HISTORYSERVER" + }, + { + "name" : "APP_TIMELINE_SERVER" + }, + { + "name" : "YARN_CLIENT" + }, + { + "name" : "MAPREDUCE2_CLIENT" + }, + { + "name" : "GANGLIA_SERVER" + }, + { + "name" : "NAGIOS_SERVER" + } + ], + "cardinality" : "1" + } + ], + "Blueprints" : { + "blueprint_name" : "single-node-blueprint1", + "stack_name" : "HDP", + "stack_version" : "2.1" + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b42c3c9b/dev-support/docker/docker/blueprints/single-node-hostmapping1.json ---------------------------------------------------------------------- diff --git a/dev-support/docker/docker/blueprints/single-node-hostmapping1.json b/dev-support/docker/docker/blueprints/single-node-hostmapping1.json new file mode 100644 index 0000000..dc74cd8 --- /dev/null +++ b/dev-support/docker/docker/blueprints/single-node-hostmapping1.json @@ -0,0 +1,14 @@ +{ + "blueprint": "single-node-blueprint1", + "default_password": "changeme", + "host_groups": [ + { + "hosts": [ + { + "fqdn": "node1.mydomain.com" + } + ], + "name": "host_group_1" + } + ] +}
