http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/main/python/ambari_server/serverSetup_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverSetup_linux.py b/ambari-server/src/main/python/ambari_server/serverSetup_linux.py deleted file mode 100644 index b5436e0..0000000 --- a/ambari-server/src/main/python/ambari_server/serverSetup_linux.py +++ /dev/null @@ -1,795 +0,0 @@ -#!/usr/bin/env python - -''' -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -''' -import glob -import optparse - -import os -import re -import socket - -from ambari_commons.exceptions import * -from ambari_commons.logging_utils import * -from ambari_commons.os_linux import run_os_command -from ambari_server.dbConfiguration_linux import SERVICE_CMD, PG_HBA_CONF_FILE_BACKUP -from ambari_server.serverConfiguration import * -from ambari_server.serverConfiguration_linux import JAVA_SHARE_PATH -from ambari_server.setupSecurity import * -from ambari_server.userInput import get_YN_input, get_validated_string_input -from ambari_server import utils - -# selinux commands -GET_SE_LINUX_ST_CMD = utils.locate_file('sestatus', '/usr/sbin') -SE_SETENFORCE_CMD = "setenforce 0" -SE_STATUS_DISABLED = "disabled" -SE_STATUS_ENABLED = "enabled" -SE_MODE_ENFORCING = "enforcing" -SE_MODE_PERMISSIVE = "permissive" - -# Non-root user setup commands -NR_USER_COMMENT = "Ambari user" -NR_GET_OWNER_CMD = 'stat -c "%U" {0}' -NR_USERADD_CMD = 'useradd -M --comment "{1}" ' \ - '--shell %s -d /var/lib/ambari-server/keys/ {0}' % utils.locate_file('nologin', '/sbin') -NR_SET_USER_COMMENT_CMD = 'usermod -c "{0}" {1}' - -NR_USER_CHANGE_PROMPT = "Ambari-server daemon is configured to run under user '{0}'. Change this setting [y/n] (n)? " -NR_USER_CUSTOMIZE_PROMPT = "Customize user account for ambari-server daemon [y/n] (n)? " -NR_DEFAULT_USER = "root" - -# jdk commands -JDK_URL_PROPERTIES = ["jdk1.7.url", "jdk1.6.url"] -JCE_URL_PROPERTIES = ["jce_policy1.7.url", "jce_policy1.6.url"] -JDK_VERSION_REs = ["(jdk.*)/jre", "Creating (jdk.*)/jre"] -JDK_CHOICE_PROMPT = "[1] - Oracle JDK 1.7\n[2] - Oracle JDK 1.6\n[3] - Custom JDK\n==============================================================================\nEnter choice ({0}): " -JDK_VALID_CHOICES = "^[123]$" -CUSTOM_JDK_NUMBER = "3" -JDK_MIN_FILESIZE = 5000 -CREATE_JDK_DIR_CMD = "/bin/mkdir -p {0}" -MAKE_FILE_EXECUTABLE_CMD = "chmod a+x {0}" - -JDK_DOWNLOAD_CMD = "curl --create-dirs -o {0} {1}" -JDK_DOWNLOAD_SIZE_CMD = "curl -I {0}" - -# use --no-same-owner when running as root to prevent uucp as the user (AMBARI-6478) -UNTAR_JDK_ARCHIVE = "tar --no-same-owner -xvf {0}" - - -#JDBC -DATABASE_INDEX = 0 -USERNAME_PATTERN = "^[a-zA-Z_][a-zA-Z0-9_\-]*$" -DATABASE_NAMES = ["postgres", "oracle", "mysql"] -DATABASE_STORAGE_NAMES = ["Database", "Service", "Database"] -DATABASE_PORTS = ["5432", "1521", "3306"] -DATABASE_DRIVER_NAMES = ["org.postgresql.Driver", "oracle.jdbc.driver.OracleDriver", "com.mysql.jdbc.Driver"] -DATABASE_CONNECTION_STRINGS = [ - "jdbc:postgresql://{0}:{1}/{2}", - "jdbc:oracle:thin:@{0}:{1}/{2}", - "jdbc:mysql://{0}:{1}/{2}"] -DATABASE_CONNECTION_STRINGS_ALT = [ - "jdbc:postgresql://{0}:{1}/{2}", - "jdbc:oracle:thin:@{0}:{1}:{2}", - "jdbc:mysql://{0}:{1}/{2}"] -ORACLE_SID_PATTERN = "jdbc:oracle:thin:@.+:.+/.+" -ORACLE_SNAME_PATTERN = "jdbc:oracle:thin:@.+:.+:.+" - -DATABASE_CLI_TOOLS = [["psql"], ["sqlplus", "sqlplus64"], ["mysql"]] -DATABASE_CLI_TOOLS_DESC = ["psql", "sqlplus", "mysql"] -DATABASE_CLI_TOOLS_USAGE = ['su -postgres --command=psql -f {0} -v username=\'"{1}"\' -v password="\'{2}\'"', - 'sqlplus {1}/{2} < {0} ', - 'mysql --user={1} --password={2} {3}<{0}'] - -MYSQL_INIT_SCRIPT = '/var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql' -DATABASE_INIT_SCRIPTS = ['/var/lib/ambari-server/resources/Ambari-DDL-Postgres-CREATE.sql', - '/var/lib/ambari-server/resources/Ambari-DDL-Oracle-CREATE.sql', - MYSQL_INIT_SCRIPT] -DATABASE_DROP_SCRIPTS = ['/var/lib/ambari-server/resources/Ambari-DDL-Postgres-DROP.sql', - '/var/lib/ambari-server/resources/Ambari-DDL-Oracle-DROP.sql', - '/var/lib/ambari-server/resources/Ambari-DDL-MySQL-DROP.sql'] - -JDBC_PROPERTIES_PREFIX = "server.jdbc.properties." -DATABASE_JDBC_PROPERTIES = [ - [ ], - [ - ["oracle.net.CONNECT_TIMEOUT", "2000"], # socket level timeout - ["oracle.net.READ_TIMEOUT", "2000"], # socket level timeout - ["oracle.jdbc.ReadTimeout", "8000"] # query fetch timeout - ], - [ ] - ] - -POSTGRES_EXEC_ARGS = "-h {0} -p {1} -d {2} -U {3} -f {4} -v username='{3}'" -ORACLE_EXEC_ARGS = "-S -L '{0}/{1}@(description=(address=(protocol=TCP)(host={2})(port={3}))(connect_data=({6}={4})))' @{5} {0}" -MYSQL_EXEC_ARGS_WITH_USER_VARS = "--host={0} --port={1} --user={2} --password={3} {4} " \ - "-e\"set @schema=\'{4}\'; set @username=\'{2}\'; source {5};\"" -MYSQL_EXEC_ARGS_WO_USER_VARS = "--force --host={0} --port={1} --user={2} --password={3} --database={4} < {5} 2> /dev/null" -MYSQL_UPGRADE_STACK_ARGS = "--host={0} --port={1} --user={2} --password={3} --database={4} " \ - "-e\"set @stackName=\'{6}\'; set @stackVersion=\'{7}\'; source {5};\"" - -ORACLE_UPGRADE_STACK_ARGS = "-S -L '{0}/{1}@(description=(address=(protocol=TCP)(host={2})(port={3}))(connect_data=({6}={4})))' @{5} {7} {8}" - -JDBC_PATTERNS = {"oracle": "*ojdbc*.jar", "mysql": "*mysql*.jar"} -DATABASE_FULL_NAMES = {"oracle": "Oracle", "mysql": "MySQL", "postgres": "PostgreSQL"} -JDBC_DB_OPTION_VALUES = ["postgres", "mysql", "oracle"] -JDBC_DB_DEFAULT_DRIVER = {"postgresql" : "postgresql-jdbc.jar", "mysql" : "mysql-connector-java.jar", "oracle" : "ojdbc6.jar"} -ORACLE_DB_ID_TYPES = ["Service Name", "SID"] - - -DEFAULT_DB_NAME = "ambari" - - -MESSAGE_ERROR_NOT_ROOT = 'Ambari-server setup should be run with root-level privileges' - -MESSAGE_CHECK_FIREWALL = 'Checking iptables...' - -class FirewallChecks(object): - def __init__(self): - - self.FIREWALL_SERVICE_NAME = "iptables" - self.SERVICE_CMD = SERVICE_CMD - self.SERVICE_SUBCMD = "status" - - def get_command(self): - return "%s %s %s" % (self.SERVICE_CMD, self.FIREWALL_SERVICE_NAME, self.SERVICE_SUBCMD) - - def check_result(self, retcode, out, err): - return retcode == 0 - - def check_iptables(self): - retcode, out, err = run_os_command(self.get_command()) - if err and len(err) > 0: - print err - if self.check_result(retcode, out, err): - print_warning_msg("%s is running. Confirm the necessary Ambari ports are accessible. " % - self.FIREWALL_SERVICE_NAME + - "Refer to the Ambari documentation for more details on ports.") - ok = get_YN_input("OK to continue [y/n] (y)? ", True) - if not ok: - raise FatalException(1, None) - - def get_running_result(self): - # To support test code. Expected ouput from run_os_command. - return (0, "", "") - - def get_stopped_result(self): - # To support test code. Expected output from run_os_command. - return (3, "", "") - - -class UbuntuFirewallChecks(FirewallChecks): - def __init__(self): - super(UbuntuFirewallChecks, self).__init__() - - self.FIREWALL_SERVICE_NAME = "ufw" - self.SERVICE_CMD = utils.locate_file('service', '/usr/sbin') - - def check_result(self, retcode, out, err): - # On ubuntu, the status command returns 0 whether running or not - return out and len(out) > 0 and out.strip() != "ufw stop/waiting" - - def get_running_result(self): - # To support test code. Expected ouput from run_os_command. - return (0, "ufw start/running", "") - - def get_stopped_result(self): - # To support test code. Expected output from run_os_command. - return (0, "ufw stop/waiting", "") - - -class Fedora18FirewallChecks(FirewallChecks): - def __init__(self): - self.FIREWALL_SERVICE_NAME = "firewalld.service" - - def get_command(self): - return "systemctl is-active firewalld.service" - - -class OpenSuseFirewallChecks(FirewallChecks): - def __init__(self): - self.FIREWALL_SERVICE_NAME = "SuSEfirewall2" - - def get_command(self): - return "/sbin/SuSEfirewall2 status" - - -def get_firewall_object(): - if OS_TYPE == OSConst.OS_UBUNTU: - return UbuntuFirewallChecks() - elif OS_TYPE == OSConst.OS_FEDORA and int(OS_VERSION) >= 18: - return Fedora18FirewallChecks() - elif OS_TYPE == OSConst.OS_OPENSUSE: - return OpenSuseFirewallChecks() - else: - return FirewallChecks() - - -def get_firewall_object_types(): - # To support test code, so tests can loop through the types - return (FirewallChecks, - UbuntuFirewallChecks, - Fedora18FirewallChecks, - OpenSuseFirewallChecks) - - -def os_check_firewall(): - return get_firewall_object().check_iptables() - - -# -# Checks SELinux -# -def check_selinux(): - try: - retcode, out, err = run_os_command(GET_SE_LINUX_ST_CMD) - se_status = re.search('(disabled|enabled)', out).group(0) - print "SELinux status is '" + se_status + "'" - if se_status == SE_STATUS_DISABLED: - return 0 - else: - try: - se_mode = re.search('(enforcing|permissive)', out).group(0) - except AttributeError: - err = "Error determining SELinux mode. Exiting." - raise FatalException(1, err) - print "SELinux mode is '" + se_mode + "'" - if se_mode == SE_MODE_ENFORCING: - print "Temporarily disabling SELinux" - run_os_command(SE_SETENFORCE_CMD) - print_warning_msg( - "SELinux is set to 'permissive' mode and temporarily disabled.") - ok = get_YN_input("OK to continue [y/n] (y)? ", True) - if not ok: - raise FatalException(1, None) - return 0 - except OSError: - print_warning_msg("Could not run {0}: OK".format(GET_SE_LINUX_ST_CMD)) - return 0 - -def disable_security_enhancements(): - print 'Checking SELinux...' - err = '' - retcode = check_selinux() - if not retcode == 0: - err = 'Failed to disable SELinux. Exiting.' - return (retcode, err) - - -# -# User account creation -# - -def os_create_custom_user(): - user = get_validated_string_input( - "Enter user account for ambari-server daemon (root):", - "root", - "^[a-z_][a-z0-9_-]{1,31}$", - "Invalid username.", - False - ) - - print_info_msg("Trying to create user {0}".format(user)) - command = NR_USERADD_CMD.format(user, NR_USER_COMMENT) - retcode, out, err = run_os_command(command) - if retcode == 9: # 9 = username already in use - print_info_msg("User {0} already exists, " - "skipping user creation".format(user)) - - elif retcode != 0: # fail - print_warning_msg("Can't create user {0}. Command {1} " - "finished with {2}: \n{3}".format(user, command, retcode, err)) - return retcode, None - - print_info_msg("User configuration is done.") - return 0, user - - -# -# JDK Setup -# - -def os_install_jdk(java_inst_file, java_home_dir): - print "Installing JDK to {0}".format(java_home_dir) - retcode, out, err = run_os_command(CREATE_JDK_DIR_CMD.format(java_home_dir)) - savedPath = os.getcwd() - os.chdir(java_home_dir) - - if java_inst_file.endswith(".bin"): - retcode, out, err = run_os_command(MAKE_FILE_EXECUTABLE_CMD.format(java_inst_file)) - retcode, out, err = run_os_command(java_inst_file + ' -noregister') - elif java_inst_file.endswith(".gz"): - retcode, out, err = run_os_command(UNTAR_JDK_ARCHIVE.format(java_inst_file)) - else: - err = "JDK installation failed.Unknown file mask." - raise FatalException(1, err) - - os.chdir(savedPath) - - if retcode != 0: - err = "Installation of JDK returned exit code %s" % retcode - raise FatalException(retcode, err) - - print "Successfully installed JDK to {0}".format(java_home_dir) - return (retcode, out) - -def os_ensure_java_home_env_var_is_set(java_home_var): - if not os.environ.has_key(JAVA_HOME) or os.environ[JAVA_HOME] != java_home_var: - os.system("SETX {0} {1} /M".format(JAVA_HOME, java_home_var)) - os.environ[JAVA_HOME] = java_home_var - pass - - -# -# JDBC Setup -# - -def os_check_jdbc_options(options): - return (options.jdbc_driver is not None and options.jdbc_db is not None) - -#Check if required jdbc drivers present -def os_find_jdbc_driver(args): - if args.dbms in JDBC_PATTERNS.keys(): - drivers = [] - drivers.extend(glob.glob(JAVA_SHARE_PATH + os.sep + JDBC_PATTERNS[args.dbms])) - if drivers: - return drivers - return -1 - return 0 - -def os_setup_jdbc_drivers(args): - result = os_find_jdbc_driver(args) - - msg = 'Before starting Ambari Server, ' \ - 'you must copy the {0} JDBC driver JAR file to {1}.'.format( - DATABASE_FULL_NAMES[args.dbms], - JAVA_SHARE_PATH) - - if result == -1: - if SILENT: - print_error_msg(msg) - raise FatalException(-1, msg) - else: - print_warning_msg(msg) - raw_input(PRESS_ENTER_MSG) - result = os_find_jdbc_driver(args) - if result == -1: - print_error_msg(msg) - raise FatalException(-1, msg) - - # Check if selected RDBMS requires drivers to copy - if type(result) is not int: - print 'Copying JDBC drivers to server resources...' - try: - resources_dir = args[RESOURCES_DIR_PROPERTY] - except KeyError: - print_error_msg("There is no value for " + RESOURCES_DIR_PROPERTY + "in " + AMBARI_PROPERTIES_FILE) - return -1 - - db_name = DATABASE_FULL_NAMES[args.dbms].lower() - jdbc_symlink = os.path.join(resources_dir, db_name + "-jdbc-driver.jar") - db_default_driver_path = os.path.join(JAVA_SHARE_PATH, JDBC_DB_DEFAULT_DRIVER[db_name]) - - if os.path.lexists(jdbc_symlink): - os.remove(jdbc_symlink) - - copy_status = copy_files(result, resources_dir) - - if not copy_status == 0: - raise FatalException(-1, "Failed to copy JDBC drivers to server resources") - - if db_default_driver_path in result: - os.symlink(os.path.join(resources_dir, JDBC_DB_DEFAULT_DRIVER[db_name]), jdbc_symlink) - -def os_load_default_db_properties(args): - args.persistence_type = 'local' - args.dbms = DATABASE_NAMES[DATABASE_INDEX] - args.database_host = "localhost" - args.database_port = DATABASE_PORTS[DATABASE_INDEX] - args.database_name = DEFAULT_DB_NAME - args.database_username = "ambari" - args.database_password = "bigdata" - args.sid_or_sname = "sname" - pass - -# Check if jdbc user is changed -def is_jdbc_user_changed(args): - properties = get_ambari_properties() - if properties == -1: - print_error_msg("Error getting ambari properties") - return None - - previos_user = properties[JDBC_USER_NAME_PROPERTY] - new_user = args.database_username - - if previos_user and new_user: - if previos_user != new_user: - return True - else: - return False - - return None - -def os_setup_database(options): - if is_local_database(options): - os_setup_local_database(options) - else: - os_setup_remote_database(options) - -def os_setup_local_database(options): - #check if jdbc user is changed - is_user_changed = is_jdbc_user_changed(options) - - print 'Default properties detected. Using built-in database.' - os_store_local_properties(options) - - print 'Checking PostgreSQL...' - pg_status, retcode, out, err = check_postgre_up() - if not retcode == 0: - err = 'Unable to start PostgreSQL server. Status {0}. {1}.' \ - ' Exiting'.format(pg_status, err) - raise FatalException(retcode, err) - - print 'Configuring local database...' - retcode, outdata, errdata = setup_db(options) - if not retcode == 0: - err = 'Running database init script was failed. {0}. Exiting.'.format(errdata) - raise FatalException(retcode, err) - - if is_user_changed: - #remove backup for pg_hba in order to reconfigure postgres - remove_file(PG_HBA_CONF_FILE_BACKUP) - - print 'Configuring PostgreSQL...' - retcode, out, err = configure_postgres() - if not retcode == 0: - err = 'Unable to configure PostgreSQL server. {0} Exiting'.format(err) - raise FatalException(retcode, err) - pass - -def os_setup_remote_database(options): - retcode = os_store_remote_properties(options) - if retcode != 0: - err = 'Unable to save config file' - raise FatalException(retcode, err) - - os_setup_jdbc_drivers(options) - - print 'Configuring remote database connection properties...' - retcode = setup_remote_db(options) - if retcode == -1: - err = "Remote database setup aborted." - raise NonFatalException(err) - - if not retcode == 0: - err = 'Error while configuring connection properties. Exiting' - raise FatalException(retcode, err) - pass - -def os_reset_database(options): - pass - - -### Postgres ### - - -def configure_pg_hba_ambaridb_users(): - args = optparse.Values() - configure_database_username_password(args) - - with open(PG_HBA_CONF_FILE, "a") as pgHbaConf: - pgHbaConf.write("\n") - pgHbaConf.write("local all " + args.database_username + - ",mapred md5") - pgHbaConf.write("\n") - pgHbaConf.write("host all " + args.database_username + - ",mapred 0.0.0.0/0 md5") - pgHbaConf.write("\n") - pgHbaConf.write("host all " + args.database_username + - ",mapred ::/0 md5") - pgHbaConf.write("\n") - retcode, out, err = run_os_command(PG_HBA_RELOAD_CMD) - if not retcode == 0: - raise FatalException(retcode, err) - - -def configure_pg_hba_postgres_user(): - postgresString = "all postgres" - for line in fileinput.input(PG_HBA_CONF_FILE, inplace=1): - print re.sub('all\s*all', postgresString, line), - os.chmod(PG_HBA_CONF_FILE, 0644) - - -def configure_postgresql_conf(): - listenAddress = "listen_addresses = '*' #" - for line in fileinput.input(POSTGRESQL_CONF_FILE, inplace=1): - print re.sub('#+listen_addresses.*?(#|$)', listenAddress, line), - os.chmod(POSTGRESQL_CONF_FILE, 0644) - - -def configure_postgres(): - if os.path.isfile(PG_HBA_CONF_FILE): - if not os.path.isfile(PG_HBA_CONF_FILE_BACKUP): - shutil.copyfile(PG_HBA_CONF_FILE, PG_HBA_CONF_FILE_BACKUP) - else: - #Postgres has been configured before, must not override backup - print "Backup for pg_hba found, reconfiguration not required" - return 0, "", "" - configure_pg_hba_postgres_user() - configure_pg_hba_ambaridb_users() - os.chmod(PG_HBA_CONF_FILE, 0644) - configure_postgresql_conf() - #restart postgresql if already running - pg_status, retcode, out, err = get_postgre_status() - if pg_status == PG_STATUS_RUNNING: - retcode, out, err = restart_postgres() - return retcode, out, err - return 0, "", "" - - -def restart_postgres(): - print "Restarting PostgreSQL" - process = subprocess.Popen(PG_RESTART_CMD.split(' '), - stdout=subprocess.PIPE, - stdin=subprocess.PIPE, - stderr=subprocess.PIPE - ) - time.sleep(5) - result = process.poll() - if result is None: - print_info_msg("Killing restart PostgresSQL process") - process.kill() - pg_status, retcode, out, err = get_postgre_status() - # SUSE linux set status of stopped postgresql proc to unused - if pg_status == "unused" or pg_status == "stopped": - print_info_msg("PostgreSQL is stopped. Restarting ...") - retcode, out, err = run_os_command(PG_START_CMD) - return retcode, out, err - return 0, "", "" - - -# todo: check if the scheme is already exist - - -def setup_db(args): - #password access to ambari-server and mapred - configure_database_username_password(args) - dbname = args.database_name - scriptFile = args.init_script_file - username = args.database_username - password = args.database_password - - #setup DB - command = SETUP_DB_CMD[:] - command[-1] = command[-1].format(scriptFile, username, password, dbname) - - for i in range(SETUP_DB_CONNECT_ATTEMPTS): - sys.stdout.write('Connecting to local database...') - retcode, outdata, errdata = run_os_command(command) - if retcode == 0: - print 'done.' - return retcode, outdata, errdata - timeOutMsg = 'connection timed out' - if (i+1) < SETUP_DB_CONNECT_ATTEMPTS: - timeOutMsg += '...retrying (%d)' % (i+1) - print timeOutMsg - time.sleep(SETUP_DB_CONNECT_TIMEOUT) - - print 'unable to connect to database' - print_error_msg(errdata) - return retcode, outdata, errdata - - -def execute_db_script(args, file): - #password access to ambari-server and mapred - configure_database_username_password(args) - dbname = args.database_name - username = args.database_username - password = args.database_password - command = SETUP_DB_CMD[:] - command[-1] = command[-1].format(file, username, password, dbname) - retcode, outdata, errdata = run_os_command(command) - if not retcode == 0: - print errdata - return retcode - - -def check_db_consistency(args, file): - #password access to ambari-server and mapred - configure_database_username_password(args) - dbname = args.database_name - username = args.database_username - password = args.database_password - command = SETUP_DB_CMD[:] - command[-1] = command[-1].format(file, username, password, dbname) - retcode, outdata, errdata = run_os_command(command) - if not retcode == 0: - print errdata - return retcode - else: - # Assumes that the output is of the form ...\n<count> - print_info_msg("Parsing output: " + outdata) - lines = outdata.splitlines() - if (lines[-1] == '3' or lines[-1] == '0'): - return 0 - return -1 - - -def get_postgre_status(): - retcode, out, err = run_os_command(PG_ST_CMD) - try: - pg_status = re.search('(stopped|running)', out, re.IGNORECASE).group(0).lower() - except AttributeError: - pg_status = None - return pg_status, retcode, out, err - - -def check_postgre_up(): - pg_status, retcode, out, err = get_postgre_status() - if pg_status == PG_STATUS_RUNNING: - print_info_msg("PostgreSQL is running") - return pg_status, 0, out, err - else: - # run initdb only on non ubuntu systems as ubuntu does not have initdb cmd. - if OS_TYPE != OSConst.OS_UBUNTU: - print "Running initdb: This may take upto a minute." - retcode, out, err = run_os_command(PG_INITDB_CMD) - if retcode == 0: - print out - print "About to start PostgreSQL" - try: - process = subprocess.Popen(PG_START_CMD.split(' '), - stdout=subprocess.PIPE, - stdin=subprocess.PIPE, - stderr=subprocess.PIPE - ) - if OS_TYPE == OSConst.OS_SUSE: - time.sleep(20) - result = process.poll() - print_info_msg("Result of postgres start cmd: " + str(result)) - if result is None: - process.kill() - pg_status, retcode, out, err = get_postgre_status() - else: - retcode = result - else: - out, err = process.communicate() - retcode = process.returncode - if pg_status == PG_STATUS_RUNNING: - print_info_msg("Postgres process is running. Returning...") - return pg_status, 0, out, err - except (Exception), e: - pg_status, retcode, out, err = get_postgre_status() - if pg_status == PG_STATUS_RUNNING: - return pg_status, 0, out, err - else: - print_error_msg("Postgres start failed. " + str(e)) - return pg_status, retcode, out, err - -def get_validated_db_name(database_name): - return get_validated_string_input( - DATABASE_STORAGE_NAMES[DATABASE_INDEX] + " Name (" - + database_name + "): ", - database_name, - ".*", - "Invalid " + DATABASE_STORAGE_NAMES[DATABASE_INDEX] + " name.", - False - ) - -def get_validated_service_name(service_name, index): - return get_validated_string_input( - ORACLE_DB_ID_TYPES[index] + " (" + service_name + "): ", - service_name, - ".*", - "Invalid " + ORACLE_DB_ID_TYPES[index] + ".", - False - ) - -def get_pass_file_path(conf_file): - return os.path.join(os.path.dirname(conf_file), - JDBC_PASSWORD_FILENAME) - -# Store local database connection properties -def os_store_local_properties(args): - properties = get_ambari_properties() - if properties == -1: - print_error_msg("Error getting ambari properties") - return -1 - - isSecure = get_is_secure(properties) - - properties.removeOldProp(JDBC_SCHEMA_PROPERTY) - properties.removeOldProp(JDBC_HOSTNAME_PROPERTY) - properties.removeOldProp(JDBC_RCA_DRIVER_PROPERTY) - properties.removeOldProp(JDBC_RCA_URL_PROPERTY) - properties.removeOldProp(JDBC_PORT_PROPERTY) - properties.removeOldProp(JDBC_DRIVER_PROPERTY) - properties.removeOldProp(JDBC_URL_PROPERTY) - properties.process_pair(PERSISTENCE_TYPE_PROPERTY, "local") - properties.process_pair(JDBC_DATABASE_PROPERTY, args.database_name) - properties.process_pair(JDBC_USER_NAME_PROPERTY, args.database_username) - properties.process_pair(JDBC_PASSWORD_PROPERTY, - store_password_file(args.database_password, JDBC_PASSWORD_FILENAME)) - - if isSecure: - encrypted_password = encrypt_password(JDBC_RCA_PASSWORD_ALIAS, args.database_password) - if args.database_password != encrypted_password: - properties.process_pair(JDBC_PASSWORD_PROPERTY, encrypted_password) - pass - pass - - return 0 - - -# Store set of properties for remote database connection -def os_store_remote_properties(args): - properties = get_ambari_properties() - if properties == -1: - print_error_msg("Error getting ambari properties") - return -1 - - isSecure = get_is_secure(properties) - - properties.process_pair(PERSISTENCE_TYPE_PROPERTY, "remote") - - properties.process_pair(JDBC_DATABASE_PROPERTY, args.dbms) - properties.process_pair(JDBC_HOSTNAME_PROPERTY, args.database_host) - properties.process_pair(JDBC_PORT_PROPERTY, args.database_port) - properties.process_pair(JDBC_SCHEMA_PROPERTY, args.database_name) - - properties.process_pair(JDBC_DRIVER_PROPERTY, DATABASE_DRIVER_NAMES[DATABASE_INDEX]) - # fully qualify the hostname to make sure all the other hosts can connect - # to the jdbc hostname since its passed onto the agents for RCA - jdbc_hostname = args.database_host - if (args.database_host == "localhost"): - jdbc_hostname = socket.getfqdn() - - connectionStringFormat = DATABASE_CONNECTION_STRINGS - if args.sid_or_sname == "sid": - connectionStringFormat = DATABASE_CONNECTION_STRINGS_ALT - properties.process_pair(JDBC_URL_PROPERTY, - connectionStringFormat[DATABASE_INDEX].format(jdbc_hostname, args.database_port, - args.database_name)) - properties.process_pair(JDBC_USER_NAME_PROPERTY, args.database_username) - properties.process_pair(JDBC_PASSWORD_PROPERTY, - store_password_file(args.database_password, JDBC_PASSWORD_FILENAME)) - - # save any other defined properties to pass to JDBC - if DATABASE_INDEX < len(DATABASE_JDBC_PROPERTIES): - for pair in DATABASE_JDBC_PROPERTIES[DATABASE_INDEX]: - properties.process_pair(JDBC_PROPERTIES_PREFIX + pair[0], pair[1]) - - if isSecure: - encrypted_password = encrypt_password(JDBC_RCA_PASSWORD_ALIAS, args.database_password) - if encrypted_password != args.database_password: - properties.process_pair(JDBC_PASSWORD_PROPERTY, encrypted_password) - pass - - properties.process_pair(JDBC_RCA_DRIVER_PROPERTY, DATABASE_DRIVER_NAMES[DATABASE_INDEX]) - properties.process_pair(JDBC_RCA_URL_PROPERTY, - connectionStringFormat[DATABASE_INDEX].format(jdbc_hostname, args.database_port, - args.database_name)) - properties.process_pair(JDBC_RCA_USER_NAME_PROPERTY, args.database_username) - properties.process_pair(JDBC_RCA_PASSWORD_FILE_PROPERTY, - store_password_file(args.database_password, JDBC_PASSWORD_FILENAME)) - if isSecure: - encrypted_password = encrypt_password(JDBC_RCA_PASSWORD_ALIAS, args.database_password) - if encrypted_password != args.database_password: - properties.process_pair(JDBC_RCA_PASSWORD_FILE_PROPERTY, encrypted_password) - pass - - return 0
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/main/python/ambari_server/serverSetup_windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverSetup_windows.py b/ambari-server/src/main/python/ambari_server/serverSetup_windows.py deleted file mode 100644 index a906ef5..0000000 --- a/ambari-server/src/main/python/ambari_server/serverSetup_windows.py +++ /dev/null @@ -1,313 +0,0 @@ -#!/usr/bin/env python - -''' -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -''' - -import optparse -import os -import socket -import string - -from _winreg import (OpenKey, EnumValue, HKEY_LOCAL_MACHINE, KEY_READ, CloseKey, KEY_WRITE, QueryValueEx, SetValueEx, - REG_EXPAND_SZ) - -from ambari_commons import os_utils - -from ambari_commons.exceptions import * -from ambari_commons.logging_utils import * -from ambari_commons.os_windows import run_os_command, UserHelper -from ambari_server.dbConfiguration import DBMSConfig -from ambari_server.serverConfiguration import * -from ambari_server.serverConfiguration_windows import OUT_DIR -from ambari_server.userInput import get_validated_string_input - -# Non-root user setup commands -NR_USER_COMMENT = "Ambari user" -NR_GET_OWNER_CMD = 'stat -c "%U" {0}' -NR_USERADD_CMD = 'cmd /C net user {0} {1} /ADD' -NR_SET_USER_COMMENT_CMD = 'usermod -c "{0}" {1}' - -NR_USER_CHANGE_PROMPT = "Ambari-server service is configured to run under user '{0}'. Change this setting [y/n] (n)? " -NR_USER_CUSTOMIZE_PROMPT = "Customize user account for ambari-server service [y/n] (n)? " -NR_DEFAULT_USER = "NT AUTHORITY\SYSTEM" - -SERVICE_USERNAME_KEY = "TMP_AMBARI_USERNAME" -SERVICE_PASSWORD_KEY = "TMP_AMBARI_PASSWORD" - -# JDK setup choices -JDK_DEFAULT_CONFIGS = [ - JDKRelease("jdk7.67", "Oracle JDK 7.67", - "http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-7u67-windows-x64.exe";, "jdk-7u67-windows-x64.exe", - "http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip";, "UnlimitedJCEPolicyJDK7.zip", - None) -] - -JDK_VERSION_REs = ["(jdk.*)/jre", "Creating (jdk.*)/jre"] -JDK_PROMPT = "[{0}] {1}\n" -JDK_CUSTOM_CHOICE_PROMPT = "[{0}] - Custom JDK\n==============================================================================\nEnter choice ({1}): " -JDK_VALID_CHOICES = "^[{0}{1:d}]$" -CUSTOM_JDK_NUMBER = "4" -JDK_MIN_FILESIZE = 5000 -MAKE_FILE_EXECUTABLE_CMD = "chmod a+x {0}" - -JDK_DOWNLOAD_CMD = "curl --create-dirs -o {0} {1}" -JDK_DOWNLOAD_SIZE_CMD = "curl -I {0}" - -# use --no-same-owner when running as root to prevent uucp as the user (AMBARI-6478) -UNTAR_JDK_ARCHIVE = "tar --no-same-owner -xvf {0}" - - -#JDBC -USERNAME_PATTERN = "^[a-zA-Z_][a-zA-Z0-9_\-]*$" -DATABASE_DBMS = "sqlserver" -DATABASE_NAME = "ambari" -DATABASE_SERVER = "localhost\\\\SQLEXPRESS" -DATABASE_DRIVER_NAME = "com.microsoft.sqlserver.jdbc.SQLServerDriver" - -METRICS_DATABASE_NAME = "HadoopMetrics" - -JDBC_PATTERNS = {"sqlserver": "sqljdbc*.jar"} -DATABASE_FULL_NAMES = {"sqlserver": "SQL Server"} -JDBC_DB_OPTION_VALUES = ["sqlserver"] -JDBC_DB_DEFAULT_DRIVER = {"sqlserver" : "sqljdbc4.jar"} - - -ERROR_NOT_ROOT = 'Ambari-server setup should be run with administrator-level privileges' - -MESSAGE_CHECK_FIREWALL = 'Checking firewall status...' - -def os_check_firewall(): - out = run_powershell_script(CHECK_FIREWALL_SCRIPT) - if out[0] != 0: - print_warning_msg("Unable to check firewall status:{0}".format(out[2])) - return False - profiles_status = [i for i in out[1].split("\n") if not i == ""] - if "1" in profiles_status: - enabled_profiles = [] - if profiles_status[0] == "1": - enabled_profiles.append("DomainProfile") - if profiles_status[1] == "1": - enabled_profiles.append("StandardProfile") - if profiles_status[2] == "1": - enabled_profiles.append("PublicProfile") - print_warning_msg("Following firewall profiles enabled:{0}. Make sure that firewall properly configured.".format(",".join(enabled_profiles))) - return False - return True - -# No security enhancements in Windows -def disable_security_enhancements(): - retcode = 0 - err = '' - return (retcode, err) - - -# -# User account creation -# - -def os_create_custom_user(): - user = get_validated_string_input( - "Enter user account for ambari-server service ({0}):".format(NR_DEFAULT_USER), - NR_DEFAULT_USER, None, - "Invalid username.", - False - ) - if user == NR_DEFAULT_USER: - return 0, user - password = get_validated_string_input("Enter password for user {0}:".format(user), "", None, "Password", True, False) - - uh = UserHelper() - - status, message = uh.create_user(user,password) - if status == UserHelper.USER_EXISTS: - print_info_msg("User {0} already exists, make sure that you typed correct password for user, " - "skipping user creation".format(user)) - - elif status == UserHelper.ACTION_FAILED: # fail - print_warning_msg("Can't create user {0}. Failed with message {1}".format(user, message)) - return UserHelper.ACTION_FAILED, None - - # setting SeServiceLogonRight to user - - status, message = uh.add_user_privilege(user, 'SeServiceLogonRight') - if status == UserHelper.ACTION_FAILED: - print_warning_msg("Can't add SeServiceLogonRight to user {0}. Failed with message {1}".format(user, message)) - return UserHelper.ACTION_FAILED, None - - print_info_msg("User configuration is done.") - print_warning_msg("When using non SYSTEM user make sure that your user have read\write access to log directories and " - "all server directories. In case of integrated authentication for SQL Server make sure that your " - "user properly configured to use ambari and metric database.") - #storing username and password in os.environ temporary to pass them to service - os.environ[SERVICE_USERNAME_KEY] = user - os.environ[SERVICE_PASSWORD_KEY] = password - return 0, user - - -# -# JDK Setup -# -def populate_jdk_configs(properties, jdk_num): - if properties.has_key(JDK_RELEASES): - jdk_names = properties[JDK_RELEASES].split(',') - jdks = [] - for jdk_name in jdk_names: - jdkR = JDKRelease.from_properties(properties, jdk_name) - jdks.append(jdkR) - else: - jdks = JDK_DEFAULT_CONFIGS - - n_config = 1 - jdk_choice_prompt = '' - jdk_choices = '' - for jdk in jdks: - jdk_choice_prompt += JDK_PROMPT.format(n_config, jdk.desc) - jdk_choices_tmp = '{0}{1:d}'.format(jdk_choices, n_config) - jdk_choices = jdk_choices_tmp - n_config += 1 - - jdk_choice_prompt += JDK_CUSTOM_CHOICE_PROMPT.format(n_config, jdk_num) - jdk_valid_choices = JDK_VALID_CHOICES.format(jdk_choices, n_config) - - return (jdks, jdk_choice_prompt, jdk_valid_choices, n_config) - - -def os_install_jdk(java_inst_file, java_home_dir): - print "Installing JDK to {0}".format(java_home_dir) - - if not os.path.exists(java_home_dir): - os.makedirs(java_home_dir) - - if java_inst_file.endswith(".exe"): - (dirname, filename) = os.path.split(java_inst_file) - installLogFilePath = os.path.join(OUT_DIR, filename + "-install.log") - #jre7u67.exe /s INSTALLDIR=<dir> STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-server\\jre7u67.exe-install.log - installCmd = [ - java_inst_file, - "/s", - "INSTALLDIR=" + java_home_dir, - "STATIC=1", - "WEB_JAVA=0", - "/L", - installLogFilePath - ] - retcode, out, err = run_os_command(installCmd) - #TODO: support .msi file installations - #msiexec.exe jre.msi /s INSTALLDIR=<dir> STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-server\\jre7u67-install.log ? - else: - err = "JDK installation failed.Unknown file mask." - raise FatalException(1, err) - - if retcode == 1603: - # JDK already installed - print "JDK already installed in {0}".format(java_home_dir) - retcode = 0 - else: - if retcode != 0: - err = "Installation of JDK returned exit code %s" % retcode - raise FatalException(retcode, err) - - print "Successfully installed JDK to {0}".format(java_home_dir) - - # Don't forget to adjust the JAVA_HOME env var - - return (retcode, out) - -def os_ensure_java_home_env_var_is_set(java_home_var): - if not os.environ.has_key(JAVA_HOME) or os.environ[JAVA_HOME] != java_home_var: - java_home_var_val = java_home_var.replace('\\\\', '\\') - os.system("SETX {0} {1} /M".format(JAVA_HOME, java_home_var_val)) - os.environ[JAVA_HOME] = java_home_var - pass - -# -# JDBC Setup -# - -def os_check_jdbc_options(options): - #Only SQL Server supported, no point in checking options.jdbc_db - return (options.jdbc_driver is not None) - -def os_setup_jdbc_drivers(args): - properties = get_ambari_properties() - if properties == -1: - print_error_msg("Error getting ambari properties") - return -1 - - #Only support SQL Server - dbms = DBMSConfig.create(args, properties) - if dbms.ensure_jdbc_driver_installed(args, properties): - # Now save the properties file - update_properties(properties) - pass - -def os_setup_database(options): - properties = get_ambari_properties() - if properties == -1: - raise FatalException(-1, "Error getting ambari properties") - - #Ensure the default database host is set - options.default_database_host = "localhost\\SQLEXPRESS" - - #Only support SQL Server - dbmsAmbari = DBMSConfig.create(options, properties, "Ambari") - resultA = dbmsAmbari.configure_database(options, properties) - - #By default, use the same server for Metrics - options.default_database_host = dbmsAmbari.database_host - - dbmsMetrics = DBMSConfig.create(options, properties, "Metrics") - resultM = dbmsMetrics.configure_database(options, properties) - - # Now save the properties file - if resultA or resultM: - update_properties(properties) - - dbmsAmbari.setup_database() - dbmsMetrics.setup_database() - -def os_reset_database(options): - properties = get_ambari_properties() - if properties == -1: - raise FatalException(-1, "Error getting ambari properties") - - if not (properties.getPropertyDict().has_key(JDBC_URL_PROPERTY) and - properties.getPropertyDict().has_key(JDBC_RCA_URL_PROPERTY) and - properties.getPropertyDict().has_key(JDBC_METRICS_URL_PROPERTY)): - raise FatalException(-1, "Ambari Server not set up yet. Nothing to reset.") - - empty_options = optparse.Values() - empty_options.silent = options.silent - empty_options.database_host = "" - empty_options.database_port = "" - empty_options.database_name = "" - empty_options.database_windows_auth = False - empty_options.database_username = "" - empty_options.database_password = "" - empty_options.init_db_script_file = "" - empty_options.cleanup_db_script_file = "" - empty_options.init_metrics_db_script_file = "" - empty_options.cleanup_metrics_db_script_file = "" - - #Only support SQL Server - dbmsAmbari = DBMSConfig.create(empty_options, properties, "Ambari") - dbmsAmbari.reset_database() - - dbmsMetrics = DBMSConfig.create(empty_options, properties, "Metrics") - dbmsMetrics.reset_database() - pass http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/main/python/ambari_server/setupSecurity.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/setupSecurity.py b/ambari-server/src/main/python/ambari_server/setupSecurity.py index 57a3231..fd1ee8a 100644 --- a/ambari-server/src/main/python/ambari_server/setupSecurity.py +++ b/ambari-server/src/main/python/ambari_server/setupSecurity.py @@ -19,16 +19,32 @@ limitations under the License. ''' import datetime import fileinput +import os import random +import re +import shutil import socket import stat +import string import sys +import tempfile +import time import urllib2 -from ambari_commons.exceptions import * -from serverConfiguration import * -from setupActions import * -from userInput import * +from ambari_commons.exceptions import FatalException, NonFatalException +from ambari_commons.logging_utils import print_error_msg, print_info_msg, print_warning_msg, SILENT +from ambari_commons.os_utils import copy_file, remove_file, search_file, set_file_permissions, is_valid_filepath, \ + is_root, run_os_command +from ambari_server.serverConfiguration import find_properties_file, get_ambari_properties, get_value_from_properties, \ + get_conf_dir, get_full_ambari_classpath, read_ambari_user, find_jdk, get_prompt_default, update_properties_2, \ + configDefaults, \ + BOOTSTRAP_DIR_PROPERTY, GET_FQDN_SERVICE_URL, \ + JDBC_USE_INTEGRATED_AUTH_PROPERTY, JDBC_PASSWORD_PROPERTY, JDBC_PASSWORD_FILENAME, \ + JDBC_RCA_PASSWORD_ALIAS, JDBC_RCA_PASSWORD_FILE_PROPERTY, \ + JDBC_METRICS_USE_INTEGRATED_AUTH_PROPERTY, JDBC_METRICS_PASSWORD_PROPERTY, \ + JDBC_METRICS_PASSWORD_ALIAS, JDBC_METRICS_PASSWORD_FILENAME, BLIND_PASSWORD +from ambari_server.setupActions import SETUP_ACTION, LDAP_SETUP_ACTION +from ambari_server.userInput import get_validated_filepath_input, get_validated_string_input, get_YN_input SSL_PASSWORD_FILE = "pass.txt" @@ -40,36 +56,25 @@ EXPRT_KSTR_CMD = "openssl pkcs12 -export -in '{0}' -inkey '{1}' -certfile '{0}' CHANGE_KEY_PWD_CND = 'openssl rsa -in {0} -des3 -out {0}.secured -passout pass:{1}' GET_CRT_INFO_CMD = 'openssl x509 -dates -subject -in {0}' -#keytool commands -keytool_bin = "keytool" -if OSCheck.is_windows_family(): - keytool_bin = "keytool.exe" - -KEYTOOL_IMPORT_CERT_CMD = "{0}" + os.sep + "bin" + os.sep + keytool_bin + " -import -alias '{1}' -storetype '{2}' -file '{3}' -storepass '{4}' -noprompt" -KEYTOOL_DELETE_CERT_CMD = "{0}" + os.sep + "bin" + os.sep + keytool_bin + " -delete -alias '{1}' -storepass '{2}' -noprompt" +#keytool command +KEYTOOL_IMPORT_CERT_CMD = "{0}" + os.sep + "bin" + os.sep + configDefaults.keytool_bin + " -import -alias '{1}' -storetype '{2}' -file '{3}' -storepass '{4}' -noprompt" +KEYTOOL_DELETE_CERT_CMD = "{0}" + os.sep + "bin" + os.sep + configDefaults.keytool_bin + " -delete -alias '{1}' -storepass '{2}' -noprompt" KEYTOOL_KEYSTORE = " -keystore '{0}'" -java_bin = "java" -if OSCheck.is_windows_family(): - java_bin = "java.exe" - -SECURITY_PROVIDER_GET_CMD = "{0}" + os.sep + "bin" + os.sep + java_bin + " -cp {1}" +\ - os.pathsep + "{2} " +\ +SECURITY_PROVIDER_GET_CMD = "{0}" + os.sep + configDefaults.JAVA_EXE_SUBPATH + " -cp {1} " +\ "org.apache.ambari.server.security.encryption" +\ - ".CredentialProvider GET {3} {4} {5} " +\ - "> " + SERVER_OUT_FILE + " 2>&1" + ".CredentialProvider GET {2} {3} {4} " +\ + "> " + configDefaults.SERVER_OUT_FILE + " 2>&1" -SECURITY_PROVIDER_PUT_CMD = "{0}" + os.sep + "bin" + os.sep + java_bin + " -cp {1}" +\ - os.pathsep + "{2} " +\ +SECURITY_PROVIDER_PUT_CMD = "{0}" + os.sep + configDefaults.JAVA_EXE_SUBPATH + " -cp {1} " +\ "org.apache.ambari.server.security.encryption" +\ - ".CredentialProvider PUT {3} {4} {5} " +\ - "> " + SERVER_OUT_FILE + " 2>&1" + ".CredentialProvider PUT {2} {3} {4} " +\ + "> " + configDefaults.SERVER_OUT_FILE + " 2>&1" -SECURITY_PROVIDER_KEY_CMD = "{0}" + os.sep + "bin" + os.sep + java_bin + " -cp {1}" +\ - os.pathsep + "{2} " +\ +SECURITY_PROVIDER_KEY_CMD = "{0}" + os.sep + configDefaults.JAVA_EXE_SUBPATH + " -cp {1}" +\ "org.apache.ambari.server.security.encryption" +\ - ".MasterKeyServiceImpl {3} {4} {5} " +\ - "> " + SERVER_OUT_FILE + " 2>&1" + ".MasterKeyServiceImpl {2} {3} {4} " +\ + "> " + configDefaults.SERVER_OUT_FILE + " 2>&1" SSL_KEY_DIR = 'security.server.keys_dir' SSL_API_PORT = 'client.api.ssl.port' @@ -521,7 +526,7 @@ def read_passwd_for_alias(alias, masterKey=""): if jdk_path is None: print_error_msg("No JDK found, please run the \"setup\" " "command to install a JDK automatically or install any " - "JDK manually to " + JDK_INSTALL_DIR) + "JDK manually to " + configDefaults.JDK_INSTALL_DIR) return 1 tempFileName = "ambari.passwd" @@ -536,8 +541,7 @@ def read_passwd_for_alias(alias, masterKey=""): if masterKey is None or masterKey == "": masterKey = "None" - command = SECURITY_PROVIDER_GET_CMD.format(jdk_path, - get_conf_dir(), get_ambari_classpath(), alias, tempFilePath, masterKey) + command = SECURITY_PROVIDER_GET_CMD.format(jdk_path, get_full_ambari_classpath(), alias, tempFilePath, masterKey) (retcode, stdout, stderr) = run_os_command(command) print_info_msg("Return code from credential provider get passwd: " + str(retcode)) @@ -574,14 +578,13 @@ def save_passwd_for_alias(alias, passwd, masterKey=""): if jdk_path is None: print_error_msg("No JDK found, please run the \"setup\" " "command to install a JDK automatically or install any " - "JDK manually to " + JDK_INSTALL_DIR) + "JDK manually to " + configDefaults.JDK_INSTALL_DIR) return 1 if masterKey is None or masterKey == "": masterKey = "None" - command = SECURITY_PROVIDER_PUT_CMD.format(jdk_path, get_conf_dir(), - get_ambari_classpath(), alias, passwd, masterKey) + command = SECURITY_PROVIDER_PUT_CMD.format(jdk_path, get_full_ambari_classpath(), alias, passwd, masterKey) (retcode, stdout, stderr) = run_os_command(command) print_info_msg("Return code from credential provider save passwd: " + str(retcode)) @@ -597,7 +600,7 @@ def get_is_persisted(properties): return (isPersisted, masterKeyFile) def get_credential_store_location(properties): - store_loc = properties[SECURITY_KEYS_DIR] + store_loc = get_value_from_properties(properties, SECURITY_KEYS_DIR, "") if store_loc is None or store_loc == "": store_loc = "/var/lib/ambari-server/keys/credentials.jceks" else: @@ -605,9 +608,9 @@ def get_credential_store_location(properties): return store_loc def get_master_key_location(properties): - keyLocation = properties[SECURITY_MASTER_KEY_LOCATION] + keyLocation = get_value_from_properties(properties, SECURITY_MASTER_KEY_LOCATION, "") if keyLocation is None or keyLocation == "": - keyLocation = properties[SECURITY_KEYS_DIR] + keyLocation = get_value_from_properties(properties, SECURITY_KEYS_DIR, "") return keyLocation def get_original_master_key(properties): @@ -679,10 +682,9 @@ def save_master_key(master_key, key_location, persist=True): if jdk_path is None: print_error_msg("No JDK found, please run the \"setup\" " "command to install a JDK automatically or install any " - "JDK manually to " + JDK_INSTALL_DIR) + "JDK manually to " + configDefaults.JDK_INSTALL_DIR) return 1 - command = SECURITY_PROVIDER_KEY_CMD.format(jdk_path, - get_ambari_classpath(), get_conf_dir(), master_key, key_location, persist) + command = SECURITY_PROVIDER_KEY_CMD.format(jdk_path, get_full_ambari_classpath(), master_key, key_location, persist) (retcode, stdout, stderr) = run_os_command(command) print_info_msg("Return code from credential provider save KEY: " + str(retcode)) @@ -722,20 +724,35 @@ def adjust_directory_permissions(ambari_user): bootstrap_dir = get_value_from_properties(properties, BOOTSTRAP_DIR_PROPERTY) print_info_msg("Cleaning bootstrap directory ({0}) contents...".format(bootstrap_dir)) shutil.rmtree(bootstrap_dir, True) #Ignore the non-existent dir error - os.makedirs(bootstrap_dir) + + #makedirs may fail with "Access denied" if executed right after rmtree. The only known good solution is a delayed retry. + retry_makedirs = 0 + while True: + try: + os.makedirs(bootstrap_dir) + except OSError as e: + time.sleep(0.01) #Unnoticeable at this scale + retry_makedirs += 1 + if retry_makedirs == 3: + raise + continue + break + + ownership_list = configDefaults.NR_ADJUST_OWNERSHIP_LIST + # Add master key and credential store if exists keyLocation = get_master_key_location(properties) masterKeyFile = search_file(SECURITY_MASTER_KEY_FILENAME, keyLocation) if masterKeyFile: - NR_ADJUST_OWNERSHIP_LIST.append((masterKeyFile, MASTER_KEY_FILE_PERMISSIONS, "{0}", "{0}", False)) + ownership_list.append((masterKeyFile, configDefaults.MASTER_KEY_FILE_PERMISSIONS, "{0}", "{0}", False)) credStoreFile = get_credential_store_location(properties) if os.path.exists(credStoreFile): - NR_ADJUST_OWNERSHIP_LIST.append((credStoreFile, CREDENTIALS_STORE_FILE_PERMISSIONS, "{0}", "{0}", False)) - trust_store_location = properties[SSL_TRUSTSTORE_PATH_PROPERTY] + ownership_list.append((credStoreFile, configDefaults.CREDENTIALS_STORE_FILE_PERMISSIONS, "{0}", "{0}", False)) + trust_store_location = get_value_from_properties(properties, SSL_TRUSTSTORE_PATH_PROPERTY, None) if trust_store_location: - NR_ADJUST_OWNERSHIP_LIST.append((trust_store_location, TRUST_STORE_LOCATION_PERMISSIONS, "{0}", "{0}", False)) + ownership_list.append((trust_store_location, configDefaults.TRUST_STORE_LOCATION_PERMISSIONS, "{0}", "{0}", False)) print "Adjusting ambari-server permissions and ownership..." - for pack in NR_ADJUST_OWNERSHIP_LIST: + for pack in ownership_list: file = pack[0] mod = pack[1] user = pack[2].format(ambari_user) @@ -829,7 +846,7 @@ def setup_component_https(component, command, property, alias): if jdk_path is None: err = "No JDK found, please run the \"ambari-server setup\" " \ "command to install a JDK automatically or install any " \ - "JDK manually to " + JDK_INSTALL_DIR + "JDK manually to " + configDefaults.JDK_INSTALL_DIR raise FatalException(1, err) properties = get_ambari_properties() @@ -979,7 +996,7 @@ def setup_master_key(): print_info_msg("Deleting master key file at location: " + str( masterKeyFile)) except Exception, e: - print 'ERROR: Could not remove master key file. %s' % e + print 'ERROR: Could not remove master key file. %s' % str(e) # Blow up the credential store made with previous key, if any store_file = get_credential_store_location(properties) if os.path.exists(store_file): http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/main/python/ambari_server/userInput.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/userInput.py b/ambari-server/src/main/python/ambari_server/userInput.py index 7a35831..ff4c0c7 100644 --- a/ambari-server/src/main/python/ambari_server/userInput.py +++ b/ambari-server/src/main/python/ambari_server/userInput.py @@ -22,7 +22,7 @@ import os import getpass import re -from ambari_commons.logging_utils import * +from ambari_commons.logging_utils import SILENT # http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/main/resources/custom_actions/scripts/check_host.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py index 7430ba1..f414d18 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py @@ -24,9 +24,10 @@ import os import subprocess import socket -from ambari_commons import os_utils from ambari_commons.os_check import OSCheck, OSConst +from ambari_commons.os_utils import quote_path from ambari_commons.inet_utils import download_file +from ambari_server.serverConfiguration import configDefaults from resource_management import Script, Execute, format from ambari_agent.HostInfo import HostInfo @@ -102,11 +103,8 @@ class CheckHost(Script): java64_home = config['commandParams']['java_home'] print "Java home to check: " + java64_home - java_bin = "java" - if OSCheck.is_windows_family(): - java_bin = "java.exe" - - if not os.path.isfile(os.path.join(java64_home, "bin", java_bin)): + + if not os.path.isfile(os.path.join(java64_home, configDefaults.JAVA_EXE_SUBPATH)): print "Java home doesn't exist!" java_home_check_structured_output = {"exit_code" : 1, "message": "Java home doesn't exist!"} else: @@ -152,13 +150,9 @@ class CheckHost(Script): jdbc_path = os.path.join(agent_cache_dir, jdbc_name) check_db_connection_path = os.path.join(agent_cache_dir, check_db_connection_jar_name) - java_bin = "java" - class_path_delimiter = ":" - if OSCheck.is_windows_family(): - java_bin = "java.exe" - class_path_delimiter = ";" + class_path_delimiter = os.pathsep - java_exec = os.path.join(java64_home, "bin",java_bin) + java_exec = os.path.join(java64_home, configDefaults.JAVA_EXE_SUBPATH) if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \ or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec): @@ -172,7 +166,7 @@ class CheckHost(Script): # download and install java if it doesn't exists if not os.path.isfile(java_exec): jdk_name = config['commandParams']['jdk_name'] - jdk_url = "{}/{}".format(jdk_location, jdk_name) + jdk_url = "{0}/{1}".format(jdk_location, jdk_name) jdk_download_target = os.path.join(agent_cache_dir, jdk_name) java_dir = os.path.dirname(java64_home) try: @@ -192,8 +186,8 @@ class CheckHost(Script): install_cmd = format("mkdir -p {java_dir} ; cd {java_dir} ; tar -xf {jdk_download_target} > /dev/null 2>&1") install_path = ["/bin","/usr/bin/"] elif jdk_name.endswith(".exe"): - install_cmd = "{} /s INSTALLDIR={} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format( - os_utils.quote_path(jdk_download_target), os_utils.quote_path(java64_home), + install_cmd = "{0} /s INSTALLDIR={1} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format( + quote_path(jdk_download_target), quote_path(java64_home), ) install_path = [java_dir]
