AMBARI-14939. [Ambari tarballs] optionally install required packages (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ed492292 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ed492292 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ed492292 Branch: refs/heads/branch-dev-patch-upgrade Commit: ed492292cd4fedd41ebe95ba97a73e57cceb0bae Parents: 6919586 Author: Andrew Onishuk <aonis...@hortonworks.com> Authored: Fri Feb 5 15:44:04 2016 +0200 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Fri Feb 5 15:44:04 2016 +0200 ---------------------------------------------------------------------- ambari-agent/pom.xml | 38 ++++-- .../src/main/package/dependencies.properties | 32 +++++ .../src/main/repo/install_ambari_tarball.py | 117 ++++++++++++++++++- ambari-server/pom.xml | 40 +++++-- .../src/main/package/dependencies.properties | 33 ++++++ 5 files changed, 239 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/ed492292/ambari-agent/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-agent/pom.xml b/ambari-agent/pom.xml index d049a6c..c2c993f 100644 --- a/ambari-agent/pom.xml +++ b/ambari-agent/pom.xml @@ -43,10 +43,7 @@ <jinja.install.dir>/usr/lib/ambari-agent/lib/ambari_jinja2</jinja.install.dir> <simplejson.install.dir>/usr/lib/ambari-agent/lib/ambari_simplejson</simplejson.install.dir> <lib.dir>/usr/lib/ambari-agent/lib</lib.dir> - <python.ver>python >= 2.6</python.ver> - <deb.python.ver>python (>= 2.6)</deb.python.ver> <deb.architecture>amd64</deb.architecture> - <deb.dependency.list>openssl, zlibc, ${deb.python.ver}</deb.dependency.list> <ambari.server.module>../ambari-server</ambari.server.module> <target.cache.dir>${project.build.directory}/cache/</target.cache.dir> <resource.keeper.script>${ambari.server.module}/src/main/python/ambari_server/resourceFilesKeeper.py</resource.keeper.script> @@ -140,6 +137,23 @@ </plugin> <plugin> <groupId>org.codehaus.mojo</groupId> + <artifactId>properties-maven-plugin</artifactId> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>read-project-properties</goal> + </goals> + <configuration> + <files> + <file>${basedir}/src/main/package/dependencies.properties</file> + </files> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> <artifactId>rpm-maven-plugin</artifactId> <version>2.1.4</version> <executions> @@ -156,10 +170,7 @@ <group>Development</group> <description>Maven Recipe: RPM Package.</description> <requires> - <require>openssl</require> - <require>rpm-python</require> - <require>zlib</require> - <require>${python.ver}</require> + <require>${rpm.dependency.list}</require> </requires> <postinstallScriptlet> <scriptFile>src/main/package/rpm/postinstall.sh</scriptFile> @@ -355,6 +366,19 @@ <resource> <directory>${basedir}/src/main/package/deb/control</directory> </resource> + <resource> + <directory>${basedir}/src/main/package</directory> + <includes> + <include>dependencies.properties</include> + </includes> + </resource> + <resource> + <directory>${project.basedir}/../ambari-common/src/main/python/ambari_commons</directory> + <includes> + <include>os_check.py</include> + <include>resources/os_family.json</include> + </includes> + </resource> </resources> </configuration> </execution> http://git-wip-us.apache.org/repos/asf/ambari/blob/ed492292/ambari-agent/src/main/package/dependencies.properties ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/package/dependencies.properties b/ambari-agent/src/main/package/dependencies.properties new file mode 100644 index 0000000..07b0b68 --- /dev/null +++ b/ambari-agent/src/main/package/dependencies.properties @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information rega4rding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Please make sure the format of the file is one of the following +# rpm.dependency.list=... +# rpm.dependency.list.[os_family_name]=... +# rpm.dependency.list.[os_family_name][starting_from_version]=... +# deb.dependency.list=... +# deb.dependency.list.[os_family_name]=... +# deb.dependency.list.[os_family_name][starting_from_version]=... +# +# Examples: +# rpm.dependency.list.suse=openssl,\nRequires: rpm-python # for all suse family os +# rpm.dependency.list.suse11=curl # for all suse family os higher or equal to 11. +# +# Such a format is respected by install_ambari_tarball.py by default, +# however should be encouraged manually in pom.xml. + +rpm.dependency.list=openssl,\nRequires: rpm-python,\nRequires: zlib,\nRequires: python >= 2.6 +deb.dependency.list=openssl, zlibc, python (>= 2.6) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ed492292/ambari-common/src/main/repo/install_ambari_tarball.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/repo/install_ambari_tarball.py b/ambari-common/src/main/repo/install_ambari_tarball.py index aa33ede..5c65b60 100644 --- a/ambari-common/src/main/repo/install_ambari_tarball.py +++ b/ambari-common/src/main/repo/install_ambari_tarball.py @@ -17,11 +17,13 @@ See the License for the specific language governing permissions and limitations under the License. ''' +import re import os import sys import logging import subprocess from optparse import OptionParser +import ConfigParser USAGE = "Usage: %prog [OPTION]... URL" DESCRIPTION = "URL should point to full tar.gz location e.g.: https://public-repo-1.hortonworks.com/something/ambari-server.tar.gz"; @@ -32,9 +34,16 @@ PREINST_SCRIPT = "preinst" PRERM_SCRIPT = "prerm" POSTINST_SCRIPT = "postinst" POSTRM_SCRIPT = "postrm" +OS_CHECK = "os_check.py" +OS_PACKAGE_DEPENDENCIES = "dependencies.properties" +OS_FAMILY_DESCRIPTION = "resources/os_family.json" +RPM_DEPENDENCIES_PROPERTY = "rpm.dependency.list" +DEB_DEPENDENCIES_PROPERTY = "deb.dependency.list" -ROOT_FOLDER_ENV_VARIABLE = "AMBARI_ROOT_FOLDER" +FILES_TO_DOWNLOAD = [PREINST_SCRIPT, PRERM_SCRIPT, POSTINST_SCRIPT, POSTRM_SCRIPT, OS_CHECK, OS_FAMILY_DESCRIPTION, OS_PACKAGE_DEPENDENCIES] +ROOT_FOLDER_ENV_VARIABLE = "AMBARI_ROOT_FOLDER" + class Utils: verbose = False @staticmethod @@ -65,26 +74,72 @@ class Utils: return out + @staticmethod + def install_package(name): + from os_check import OSCheck + + logger.info("Checking for existance of {0} dependency package".format(name)) + is_rpm = not OSCheck.is_ubuntu_family() + + if is_rpm: + is_installed_cmd = ['rpm', '-q'] + [name] + install_cmd = ['sudo', 'yum', '-y', 'install'] + [name] + else: + is_installed_cmd = ['dpkg', '-s'] + [name] + install_cmd = ['sudo', 'apt-get', '-y', 'install'] + [name] + + try: + Utils.os_call(is_installed_cmd, logoutput=False) + logger.info("Package {0} is already installed. Skipping installation.".format(name)) + except OsCallFailure: + logger.info("Package {0} is not installed. Installing it...".format(name)) + Utils.os_call(install_cmd) + + +class FakePropertiesHeader(object): + """ + Hacky class to parse properties file without sections. + see http://stackoverflow.com/questions/2819696/module-to-use-when-parsing-properties-file-in-python/2819788#2819788 + """ + FAKE_SECTION_NAME = 'section' + def __init__(self, fp): + self.fp = fp + self.sechead = '[{0}]\n'.format(FakePropertiesHeader.FAKE_SECTION_NAME) + def readline(self): + if self.sechead: + try: + return self.sechead + finally: + self.sechead = None + else: + return self.fp.readline() + class OsCallFailure(RuntimeError): pass class Installer: - def __init__(self, archive_url, root_folder, verbose): + def __init__(self, archive_url, root_folder, verbose, skip_dependencies): splited_url = archive_url.split('/') self.archive_name = splited_url[-1] self.base_url = '/'.join(splited_url[0:-1]) self.root_folder = root_folder self.verbose = verbose + self.skip_dependencies = skip_dependencies - def download_files(self): - for name in [ self.archive_name, PREINST_SCRIPT, PRERM_SCRIPT, POSTINST_SCRIPT, POSTRM_SCRIPT]: + def download_files(self, files_list): + for name in files_list: + dirname = os.path.dirname(name) + if dirname: + Utils.os_call(["mkdir", "-p", dirname]) + url = "{0}/{1}".format(self.base_url, name) logger.info("Downloading {0}".format(url)) Utils.os_call(["wget", "-O", name, url]) def run(self): - self.download_files() + self.download_files([self.archive_name] +FILES_TO_DOWNLOAD) # [self.archive_name] + + self.check_dependencies() self.run_script(PRERM_SCRIPT, ["remove"]) # in case we are upgrading self.run_script(POSTRM_SCRIPT, ["remove"]) # in case we are upgrading @@ -92,6 +147,54 @@ class Installer: self.extract_archive() self.run_script(POSTINST_SCRIPT, ["configure"]) + def check_dependencies(self): + from os_check import OSCheck + + os_family = OSCheck.get_os_family() + os_version = OSCheck.get_os_major_version() + + is_rpm = not OSCheck.is_ubuntu_family() + property_prefix = RPM_DEPENDENCIES_PROPERTY if is_rpm else DEB_DEPENDENCIES_PROPERTY + + cp = ConfigParser.SafeConfigParser() + with open(OS_PACKAGE_DEPENDENCIES) as fp: + cp.readfp(FakePropertiesHeader(fp)) + + properties = dict(cp.items(FakePropertiesHeader.FAKE_SECTION_NAME)) + + packages_string = None + postfixes = [os_family+str(ver) for ver in range(int(os_version),0,-1)+['']]+[''] + for postfix in postfixes: + property_name = property_prefix + postfix + if property_name in properties: + packages_string = properties[property_name] + break + + if packages_string is None: + err_msg = "No os dependencies found. " + if self.skip_dependencies: + logger.warn(err_msg) + else: + raise Exception(err_msg) + + packages_string = re.sub('Requires\s*:','',packages_string) + packages_string = re.sub('\\\\n','',packages_string) + packages_string = re.sub('\s','',packages_string) + packages_string = re.sub('[()]','',packages_string) + + if self.skip_dependencies: + var = raw_input("Please confirm you have the following packages installed {0} (y/n): ".format(packages_string)) + if var.lower() != "y" and var.lower() != "yes": + raise Exception("User canceled the installation.") + return + + pacakges = packages_string.split(',') + + for package in pacakges: + split_parts = re.split('[><=]', package) + package_name = split_parts[0] + Utils.install_package(package_name) + def run_script(self, script_name, args): bash_args = [] if self.verbose: @@ -112,6 +215,8 @@ class Runner: help="sets output level to more detailed") parser.add_option("-r", "--root-folder", dest="root_folder", default="/", help="root folder to install Ambari to. E.g.: /opt") + parser.add_option("-d", "--dependencies-skip", dest="skip_dependencies", action="store_true", + help="the script won't install the package dependencies. Please make sure to install them manually.") (self.options, args) = parser.parse_args() @@ -138,7 +243,7 @@ class Runner: Utils.verbose = self.options.verbose # TODO: check if ends with tar.gz? - targz_installer = TargzInstaller(self.url, self.options.root_folder, self.options.verbose) + targz_installer = TargzInstaller(self.url, self.options.root_folder, self.options.verbose, self.options.skip_dependencies) targz_installer.run() if __name__ == '__main__': http://git-wip-us.apache.org/repos/asf/ambari/blob/ed492292/ambari-server/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml index 623ddca..b500fb2 100644 --- a/ambari-server/pom.xml +++ b/ambari-server/pom.xml @@ -26,10 +26,7 @@ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <python.ver>python >= 2.6</python.ver> <!-- On centos the python xml's are inside python package --> - <python.xml.package>${python.ver}</python.xml.package> - <deb.python.ver>python (>= 2.6)</deb.python.ver> <deb.architecture>amd64</deb.architecture> - <deb.dependency.list>openssl, postgresql (>= 8.1), ${deb.python.ver}, curl</deb.dependency.list> <custom.tests>false</custom.tests> <hdpUrlForCentos6>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.1.1.0</hdpUrlForCentos6> <hdpLatestUrl>http://public-repo-1.hortonworks.com/HDP/hdp_urlinfo.json</hdpLatestUrl> @@ -195,6 +192,19 @@ <resource> <directory>${basedir}/src/main/package/deb/control</directory> </resource> + <resource> + <directory>${basedir}/src/main/package</directory> + <includes> + <include>dependencies.properties</include> + </includes> + </resource> + <resource> + <directory>${project.basedir}/../ambari-common/src/main/python/ambari_commons</directory> + <includes> + <include>os_check.py</include> + <include>resources/os_family.json</include> + </includes> + </resource> </resources> </configuration> </execution> @@ -309,6 +319,23 @@ </plugin> <plugin> <groupId>org.codehaus.mojo</groupId> + <artifactId>properties-maven-plugin</artifactId> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>read-project-properties</goal> + </goals> + <configuration> + <files> + <file>${basedir}/src/main/package/dependencies.properties</file> + </files> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> <artifactId>rpm-maven-plugin</artifactId> <version>2.1.4</version> <executions> @@ -326,10 +353,7 @@ <description>Maven Recipe: RPM Package.</description> <autoRequires>no</autoRequires> <requires> - <require>postgresql-server >= 8.1</require> - <require>openssl</require> - <require>${python.ver}</require> - <require>${python.xml.package}</require> + <require>${rpm.dependency.list}</require> </requires> <postinstallScriptlet> <scriptFile>src/main/package/rpm/postinstall.sh</scriptFile> @@ -738,7 +762,7 @@ <profile> <id>suse11</id> <properties> - <python.xml.package>python-xml</python.xml.package> + <rpm.dependency.list>${rpm.dependency.list.suse}</rpm.dependency.list> </properties> </profile> <profile> http://git-wip-us.apache.org/repos/asf/ambari/blob/ed492292/ambari-server/src/main/package/dependencies.properties ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/package/dependencies.properties b/ambari-server/src/main/package/dependencies.properties new file mode 100644 index 0000000..01244a8 --- /dev/null +++ b/ambari-server/src/main/package/dependencies.properties @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information rega4rding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Please make sure the format of the file is one of the following +# rpm.dependency.list=... +# rpm.dependency.list.[os_family_name]=... +# rpm.dependency.list.[os_family_name][starting_from_version]=... +# deb.dependency.list=... +# deb.dependency.list.[os_family_name]=... +# deb.dependency.list.[os_family_name][starting_from_version]=... +# +# Examples: +# rpm.dependency.list.suse=openssl,\nRequires: rpm-python # for all suse family os +# rpm.dependency.list.suse11=curl # for all suse family os higher or equal to 11. +# +# Such a format is respected by install_ambari_tarball.py by default, +# however should be encouraged manually in pom.xml. + +rpm.dependency.list=postgresql-server >= 8.1,\nRequires: openssl,\nRequires: python >= 2.6 +rpm.dependency.list.suse=postgresql-server >= 8.1,\nRequires: openssl,\nRequires: python-xml,\nRequires: python >= 2.6 +deb.dependency.list=openssl, postgresql (>= 8.1), python (>= 2.6), curl \ No newline at end of file
