[2/8] ambari git commit: AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components - Part 1.4
http://git-wip-us.apache.org/repos/asf/ambari/blob/49955a35/ambari-server/src/test/python/TestAmbariServer.py
--
diff --git a/ambari-server/src/test/python/TestAmbariServer.py
b/ambari-server/src/test/python/TestAmbariServer.py
index 4484458..d11e4fe 100644
--- a/ambari-server/src/test/python/TestAmbariServer.py
+++ b/ambari-server/src/test/python/TestAmbariServer.py
@@ -34,7 +34,7 @@ import StringIO
import tempfile
from unittest import TestCase
-from only_for_platform import only_for_platform, get_platform, PLATFORM_LINUX,
PLATFORM_WINDOWS
+from only_for_platform import get_platform, not_for_platform,
only_for_platform, PLATFORM_LINUX, PLATFORM_WINDOWS
if get_platform() != PLATFORM_WINDOWS:
os_distro_value = ('Suse','11','Final')
@@ -50,17 +50,35 @@ with patch("platform.linux_distribution", return_value =
os_distro_value):
with patch("glob.glob", return_value = ['/etc/init.d/postgresql-9.3']):
_ambari_server_ = __import__('ambari-server')
-from ambari_commons import Firewall, OSCheck, OSConst, FirewallChecks
+from ambari_commons import Firewall, OSCheck, OSConst
from ambari_commons.exceptions import FatalException, NonFatalException
-from ambari_commons.logging_utils import get_verbose, set_verbose,
get_silent, set_silent, get_debug_mode
-from ambari_commons.os_utils import run_os_command, search_file
+from ambari_commons.logging_utils import get_verbose, set_verbose,
get_silent, set_silent, get_debug_mode, \
+ print_info_msg, print_warning_msg, print_error_msg
+from ambari_commons.os_utils import run_os_command, search_file,
set_file_permissions, remove_file, copy_file, \
+ is_valid_filepath
+from ambari_server.dbConfiguration import DBMSConfigFactory,
check_jdbc_drivers
+from ambari_server.dbConfiguration_linux import PGConfig,
LinuxDBMSConfig, OracleConfig
from ambari_server.properties import Properties
from ambari_server.resourceFilesKeeper import ResourceFilesKeeper,
KeeperException
-from ambari_server.serverConfiguration import
AMBARI_PROPERTIES_BACKUP_FILE, AMBARI_PROPERTIES_FILE, \
- NR_USER_PROPERTY, configDefaults, \
- find_properties_file, get_ambari_properties, get_JAVA_HOME,
read_ambari_user, \
- update_ambari_properties, update_properties, update_properties_2,
write_property
-from ambari_server.userInput import get_YN_input,
get_choice_string_input
+from ambari_server.serverConfiguration import configDefaults, \
+ check_database_name_property, \
+ find_properties_file, get_ambari_classpath, get_ambari_jars,
get_ambari_properties, get_JAVA_HOME, get_share_jars, \
+ parse_properties_file, read_ambari_user, update_ambari_properties,
update_properties_2, write_property, find_jdk, \
+ AMBARI_CONF_VAR, AMBARI_SERVER_LIB, JDBC_DATABASE_PROPERTY,
JDBC_RCA_PASSWORD_FILE_PROPERTY, \
+ PERSISTENCE_TYPE_PROPERTY, JDBC_URL_PROPERTY, get_conf_dir,
JDBC_USER_NAME_PROPERTY, JDBC_PASSWORD_PROPERTY, \
+ JDBC_DATABASE_NAME_PROPERTY, OS_TYPE_PROPERTY, validate_jdk,
JDBC_POSTGRES_SCHEMA_PROPERTY, \
+ RESOURCES_DIR_PROPERTY, JDBC_RCA_PASSWORD_ALIAS,
JDBC_RCA_SCHEMA_PROPERTY, IS_LDAP_CONFIGURED, \
+ LDAP_MGR_PASSWORD_PROPERTY, LDAP_MGR_PASSWORD_ALIAS,
JDBC_PASSWORD_FILENAME, NR_USER_PROPERTY, SECURITY_KEY_IS_PERSISTED, \
+ SSL_TRUSTSTORE_PASSWORD_PROPERTY, SECURITY_IS_ENCRYPTION_ENABLED,
SSL_TRUSTSTORE_PASSWORD_ALIAS, \
+ SECURITY_MASTER_KEY_LOCATION, SECURITY_KEYS_DIR
+from ambari_server.serverSetup import check_selinux,
check_ambari_user, proceedJDBCProperties, SE_STATUS_DISABLED,
SE_MODE_ENFORCING, configure_os_settings, \
+ download_and_install_jdk, prompt_db_properties, setup, \
+ AmbariUserChecks, AmbariUserChecksLinux, AmbariUserChecksWindows,
JDKSetup, reset, is_server_runing
+from ambari_server.serverUpgrade import upgrade, upgrade_local_repo,
change_objects_owner, upgrade_stack, \
+ run_stack_upgrade, run_metainfo_upgrade, run_schema_upgrade,
move_user_custom_actions
+from ambari_server.setupSecurity import get_pass_file_path,
store_password_file, read_password, \
+ adjust_directory_permissions, get_alias_string
+from ambari_server.userInput import get_YN_input,
get_choice_string_input, get_validated_string_input
CURR_AMBARI_VERSION = "2.0.0"
@@ -74,73 +92,71 @@ class TestAmbariServer(TestCase):
sys.stdout = sys.__stdout__
- @patch.object(_ambari_server_, "configure_database_username_password")
- @patch.object(_ambari_server_, "run_os_command")
- @patch('optparse.Values')
- def test_configure_pg_hba_ambaridb_users(self, OptParseValuesMock,
- run_os_command_method,
-
co
[8/8] ambari git commit: AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components - Part 1.4
AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components - Part 1.4 +Further decomposing ambari-server.py: Moved setup, reset and upgrade into their own files +Made them OS-independent - now the same routines are being used in Windows too (save upgrade, which is not active in Windows yet) +Merged the OS-dependent code for JDBC & JDK setup, database interface creation, firewall checks +Refactored the database setup: split the supported DBMSes into distinct objects +Replaced recursive calls in the user input routines Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/49955a35 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/49955a35 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/49955a35 Branch: refs/heads/trunk Commit: 49955a35971f723dcc8d4acee57fbd9d6bec58f5 Parents: ccc58c1 Author: Florian Barca Authored: Fri Jan 23 17:36:55 2015 -0800 Committer: Florian Barca Committed: Fri Jan 23 17:36:55 2015 -0800 -- .../src/main/python/ambari_commons/firewall.py | 90 +- .../main/python/ambari_commons/inet_utils.py|6 +- .../main/python/ambari_commons/logging_utils.py | 16 +- ambari-server/conf/unix/ambari.properties | 17 +- ambari-server/conf/windows/ambari.properties|2 +- .../src/main/python/ambari-server-windows.py| 51 +- ambari-server/src/main/python/ambari-server.py | 2789 +--- .../python/ambari_server/dbConfiguration.py | 432 ++- .../ambari_server/dbConfiguration_linux.py | 1300 .../ambari_server/dbConfiguration_windows.py| 298 +- .../python/ambari_server/serverConfiguration.py | 449 ++- .../main/python/ambari_server/serverSetup.py| 1075 -- .../python/ambari_server/serverSetup_linux.py | 793 - .../python/ambari_server/serverSetup_windows.py | 300 -- .../main/python/ambari_server/serverUpgrade.py | 305 ++ .../main/python/ambari_server/setupActions.py |4 +- .../main/python/ambari_server/setupSecurity.py | 275 +- .../src/main/python/ambari_server/userInput.py | 29 +- .../custom_actions/scripts/check_host.py|2 +- .../2.1/hooks/before-ANY/scripts/setup_jdk.py |4 +- .../package/scripts/mapred_service_check.py |4 +- .../src/test/python/TestAmbariServer.py | 3081 ++ ambari-server/src/test/python/TestOSCheck.py|7 +- 23 files changed, 4782 insertions(+), 6547 deletions(-) -- http://git-wip-us.apache.org/repos/asf/ambari/blob/49955a35/ambari-common/src/main/python/ambari_commons/firewall.py -- diff --git a/ambari-common/src/main/python/ambari_commons/firewall.py b/ambari-common/src/main/python/ambari_commons/firewall.py index 76d9890..311e6d5 100644 --- a/ambari-common/src/main/python/ambari_commons/firewall.py +++ b/ambari-common/src/main/python/ambari_commons/firewall.py @@ -18,9 +18,14 @@ See the License for the specific language governing permissions and limitations under the License. ''' +import subprocess +import shlex from ambari_commons import OSCheck, OSConst +from ambari_commons.logging_utils import print_warning_msg +from ambari_commons.os_family_impl import OsFamilyImpl from ambari_commons.os_utils import run_os_command + class Firewall(object): def __init__(self): # OS info @@ -29,6 +34,16 @@ class Firewall(object): self.OS_FAMILY = OSCheck.get_os_family() def getFirewallObject(self): +pass + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class FirewallWindows(Firewall): + def getFirewallObject(self): +return WindowsFirewallChecks() + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class FirewallLinux(Firewall): + def getFirewallObject(self): if self.OS_TYPE == OSConst.OS_UBUNTU: return UbuntuFirewallChecks() elif self.OS_TYPE == OSConst.OS_FEDORA and int(self.OS_VERSION) >= 18: @@ -47,23 +62,31 @@ class FirewallChecks(object): self.returncode = None self.stdoutdata = None self.stderrdata = None +# stdout message +self.MESSAGE_CHECK_FIREWALL = 'Checking iptables...' def get_command(self): return "%s %s %s" % (self.SERVICE_CMD, self.FIREWALL_SERVICE_NAME, self.SERVICE_SUBCMD) - def check_result(self, retcode, out, err): + def check_result(self): result = False -if retcode == 3: +if self.returncode == 3: result = False -elif retcode == 0: - if "Table: filter" in out: +elif self.returncode == 0: + if "Table: filter" in self.stdoutdata: result = True return result + def run_command(self): +retcode, out, err = run_os_command(self.get_command()) +self.returncode = retcode +self.stdoutdata = out +self.stderrdata = err + def check_iptables(self): try: -
[1/8] ambari git commit: AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components - Part 1.4
Repository: ambari
Updated Branches:
refs/heads/trunk ccc58c1cd -> 49955a359
http://git-wip-us.apache.org/repos/asf/ambari/blob/49955a35/ambari-server/src/test/python/TestOSCheck.py
--
diff --git a/ambari-server/src/test/python/TestOSCheck.py
b/ambari-server/src/test/python/TestOSCheck.py
index 9ec2db3..3cebe13 100644
--- a/ambari-server/src/test/python/TestOSCheck.py
+++ b/ambari-server/src/test/python/TestOSCheck.py
@@ -38,7 +38,8 @@ with patch("platform.linux_distribution", return_value =
('Suse','11','Final')):
with patch.object(utils, "get_postgre_hba_dir"):
ambari_server = __import__('ambari-server')
- from ambari_server.serverConfiguration import update_ambari_properties
+ from ambari_server.serverConfiguration import update_ambari_properties,
configDefaults
+
class TestOSCheck(TestCase):
@patch.object(OSCheck, "os_distribution")
@@ -203,10 +204,10 @@ class TestOSCheck(TestCase):
(tf1, fn1) = tempfile.mkstemp()
(tf2, fn2) = tempfile.mkstemp()
-serverConfiguration.AMBARI_PROPERTIES_BACKUP_FILE = fn1
+configDefaults.AMBARI_PROPERTIES_BACKUP_FILE = fn1
serverConfiguration.AMBARI_PROPERTIES_FILE = fn2
-with open(serverConfiguration.AMBARI_PROPERTIES_BACKUP_FILE, 'w') as f:
+with open(configDefaults.AMBARI_PROPERTIES_BACKUP_FILE, 'w') as f:
for line in properties:
f.write(line)
[5/8] ambari git commit: AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components - Part 1.4
http://git-wip-us.apache.org/repos/asf/ambari/blob/49955a35/ambari-server/src/main/python/ambari_server/dbConfiguration_windows.py -- diff --git a/ambari-server/src/main/python/ambari_server/dbConfiguration_windows.py b/ambari-server/src/main/python/ambari_server/dbConfiguration_windows.py index ab038db..58fa7c4 100644 --- a/ambari-server/src/main/python/ambari_server/dbConfiguration_windows.py +++ b/ambari-server/src/main/python/ambari_server/dbConfiguration_windows.py @@ -21,12 +21,12 @@ limitations under the License. import os import socket import string - from ambari_commons.exceptions import FatalException from ambari_commons.logging_utils import print_info_msg, print_warning_msg from ambari_commons.os_utils import search_file, run_os_command from ambari_commons.os_windows import WinServiceController from ambari_commons.str_utils import compress_backslashes, ensure_double_backslashes +from ambari_server.dbConfiguration import AMBARI_DATABASE_NAME, DEFAULT_USERNAME, DBMSConfig, DbPropKeys, DbAuthenticationKeys from ambari_server.serverConfiguration import JDBC_DRIVER_PROPERTY, JDBC_DRIVER_PATH_PROPERTY, JDBC_URL_PROPERTY, \ JDBC_DATABASE_PROPERTY, JDBC_DATABASE_NAME_PROPERTY, \ JDBC_HOSTNAME_PROPERTY, JDBC_PORT_PROPERTY, JDBC_USE_INTEGRATED_AUTH_PROPERTY, JDBC_USER_NAME_PROPERTY, JDBC_PASSWORD_PROPERTY, \ @@ -35,59 +35,35 @@ from ambari_server.serverConfiguration import JDBC_DRIVER_PROPERTY, JDBC_DRIVER_ JDBC_RCA_HOSTNAME_PROPERTY, JDBC_RCA_PORT_PROPERTY, JDBC_RCA_USE_INTEGRATED_AUTH_PROPERTY, \ JDBC_RCA_USER_NAME_PROPERTY, JDBC_RCA_PASSWORD_FILE_PROPERTY, JDBC_RCA_PASSWORD_FILENAME, JDBC_RCA_PASSWORD_ALIAS, \ PERSISTENCE_TYPE_PROPERTY, \ - PRESS_ENTER_MSG + get_value_from_properties, configDefaults from ambari_server.setupSecurity import encrypt_password, store_password_file -from dbConfiguration import DBMSConfig, DB_STATUS_RUNNING_DEFAULT -from userInput import get_validated_string_input +from ambari_server.userInput import get_validated_string_input -#Import the SQL Server libraries # SQL Server settings -DBPATH = 'C:\\Program Files\\Microsoft SQL Server\\MSSQL12.SQLEXPRESS\\MSSQL\\DATA\\' -# DBPATH = 'C:\\Program Files\\Microsoft SQL Server\\MSSQL10_50.MSSQLSERVER\\MSSQL\\DATA\\' - -DATABASE_DBMS = "sqlserver" -DATABASE_DRIVER_NAME = "com.microsoft.sqlserver.jdbc.SQLServerDriver" -LOCAL_DATABASE_SERVER = "localhost\\SQLEXPRESS" -AMBARI_DATABASE_NAME = "ambari" - -class DbPropKeys: - def __init__(self, i_dbms_key, i_driver_key, i_server_key, i_port_key, i_db_name_key, i_db_url_key): -self.reset(i_dbms_key, i_driver_key, i_server_key, i_port_key, i_db_name_key, i_db_url_key) -pass - - def reset(self, i_dbms_key, i_driver_key, i_server_key, i_port_key, i_db_name_key, i_db_url_key): -self.dbms_key = i_dbms_key -self.driver_key = i_driver_key -self.server_key = i_server_key -self.port_key = i_port_key -self.db_name_key = i_db_name_key -self.db_url_key = i_db_url_key -pass +DATABASE_DBMS_SQLSERVER = "sqlserver" +DATABASE_DRIVER_NAME_SQLSERVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver" +DATABASE_SERVER_SQLSERVER_DEFAULT = "localhost\\SQLEXPRESS" -class AuthenticationKeys: +class SqlServerAuthenticationKeys(DbAuthenticationKeys): def __init__(self, i_integrated_auth_key, i_user_name_key, i_password_key, i_password_alias, i_password_filename): -self.reset(i_integrated_auth_key, i_user_name_key, i_password_key, i_password_alias, i_password_filename) -pass - - def reset(self, i_integrated_auth_key, i_user_name_key, i_password_key, i_password_alias, i_password_filename): self.integrated_auth_key = i_integrated_auth_key -self.user_name_key = i_user_name_key -self.password_key = i_password_key -self.password_alias = i_password_alias -self.password_filename = i_password_filename -pass +DbAuthenticationKeys.__init__(self, i_user_name_key, i_password_key, i_password_alias, i_password_filename) +# # SQL Server configuration and setup +# class SQLServerConfig(DBMSConfig): - def __init__(self, options, properties): -super(SQLServerConfig, self).__init__(options, properties) + def __init__(self, options, properties, storage_type): +super(SQLServerConfig, self).__init__(options, properties, storage_type) """ #Just load the defaults. The derived classes will be able to modify them later """ -self.dbms = DATABASE_DBMS -self.driver_name = DATABASE_DRIVER_NAME +self.dbms = DATABASE_DBMS_SQLSERVER +self.driver_class_name = DATABASE_DRIVER_NAME_SQLSERVER + +self.JDBC_DRIVER_INSTALL_MSG = 'Before starting Ambari Server, you must install the SQL Server JDBC driver.' # The values from options supersede the values from properties self.database_host = options.database_host if options.database_host is not None and options.database_host is not "" else \ @@ -98,27 +
[7/8] ambari git commit: AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components - Part 1.4
http://git-wip-us.apache.org/repos/asf/ambari/blob/49955a35/ambari-server/src/main/python/ambari-server.py -- diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py index ea072ab..f67d697 100755 --- a/ambari-server/src/main/python/ambari-server.py +++ b/ambari-server/src/main/python/ambari-server.py @@ -41,25 +41,49 @@ import random import json import base64 -from ambari_commons import OSCheck, OSConst, Firewall +from ambari_commons import OSCheck, OSConst from ambari_commons.exceptions import FatalException, NonFatalException from ambari_commons.logging_utils import get_verbose, set_verbose, get_silent, set_silent, get_debug_mode, \ - set_debug_mode, print_info_msg, print_warning_msg, print_error_msg -from ambari_commons.os_utils import is_root, run_in_shell, run_os_command, search_file + set_debug_mode, print_info_msg, print_warning_msg, print_error_msg, set_debug_mode_from_options +from ambari_commons.os_utils import is_root, run_os_command, search_file, copy_file, remove_file, \ + set_file_permissions from ambari_server.BackupRestore import main as BackupRestore_main +from ambari_server.dbConfiguration import DATABASE_NAMES, DATABASE_FULL_NAMES from ambari_server.properties import Properties from ambari_server.resourceFilesKeeper import ResourceFilesKeeper, KeeperException from ambari_server.serverConfiguration import AMBARI_PROPERTIES_FILE, configDefaults, \ - backup_file_in_temp, find_properties_file, get_conf_dir, get_value_from_properties, is_alias_string, is_local_database, \ - read_ambari_user, remove_property, update_ambari_properties, write_property, SCHEMA_UPGRADE_HELPER_CMD + backup_file_in_temp, check_database_name_property, find_jdbc_driver, find_jdk, find_properties_file, get_ambari_classpath, \ + get_ambari_properties, get_conf_dir, get_full_ambari_classpath, get_value_from_properties, is_alias_string, \ + parse_properties_file, read_ambari_user, \ + BLIND_PASSWORD, SETUP_OR_UPGRADE_MSG, JDBC_RCA_PASSWORD_ALIAS, \ + JDBC_PASSWORD_PROPERTY, JDBC_PASSWORD_FILENAME, JDBC_RCA_PASSWORD_FILE_PROPERTY, \ + GET_FQDN_SERVICE_URL, get_stack_location, IS_LDAP_CONFIGURED, LDAP_PRIMARY_URL_PROPERTY, LDAP_MGR_PASSWORD_PROPERTY, \ + LDAP_MGR_PASSWORD_ALIAS, LDAP_MGR_PASSWORD_FILENAME, LDAP_MGR_USERNAME_PROPERTY, PID_NAME, \ + read_passwd_for_alias, get_credential_store_location, get_master_key_location, get_is_secure, get_is_persisted, \ + get_original_master_key, SECURITY_PROVIDER_PUT_CMD, get_java_exe_path, SECURITY_PROVIDER_KEY_CMD, \ + SECURITY_IS_ENCRYPTION_ENABLED, SECURITY_KERBEROS_JASS_FILENAME, SECURITY_KEY_ENV_VAR_NAME, \ + SECURITY_MASTER_KEY_FILENAME, SECURITY_MASTER_KEY_LOCATION, \ + SSL_TRUSTSTORE_PASSWORD_ALIAS, SSL_TRUSTSTORE_PASSWORD_PROPERTY, SSL_TRUSTSTORE_PATH_PROPERTY, SSL_TRUSTSTORE_TYPE_PROPERTY, \ + update_debug_mode +from ambari_server.serverSetup import reset, setup, is_server_runing +from ambari_server.serverUpgrade import upgrade, upgrade_stack + +if not OSCheck.is_windows_family(): + from ambari_server.dbConfiguration_linux import PGConfig + +from ambari_server.setupActions import SETUP_ACTION, START_ACTION, STOP_ACTION, RESET_ACTION, STATUS_ACTION, \ + UPGRADE_ACTION, UPGRADE_STACK_ACTION, LDAP_SETUP_ACTION, LDAP_SYNC_ACTION, SETUP_SECURITY_ACTION, \ + REFRESH_STACK_HASH_ACTION, BACKUP_ACTION, RESTORE_ACTION, ACTION_REQUIRE_RESTART +from ambari_server.setupSecurity import adjust_directory_permissions, read_password, store_password_file, \ + remove_password_file, encrypt_password, get_truststore_password from ambari_server.userInput import get_YN_input, get_validated_string_input, get_validated_filepath_input, \ get_prompt_default -from ambari_server.utils import check_exitcode, get_postgre_hba_dir, get_postgre_running_status, locate_file, \ +from ambari_server.utils import check_exitcode, locate_file, \ looking_for_pid, save_main_pid_ex, wait_for_pid # debug settings - SERVER_START_DEBUG = False +SUSPEND_START_MODE = False # ldap settings LDAP_SYNC_ALL = False @@ -67,55 +91,10 @@ LDAP_SYNC_EXISTING = False LDAP_SYNC_USERS = None LDAP_SYNC_GROUPS = None -# OS info -OS_VERSION = OSCheck().get_os_major_version() -OS_TYPE = OSCheck.get_os_type() -OS_FAMILY = OSCheck.get_os_family() - -# action commands -SETUP_ACTION = "setup" -START_ACTION = "start" -STOP_ACTION = "stop" -RESET_ACTION = "reset" -UPGRADE_ACTION = "upgrade" -UPGRADE_STACK_ACTION = "upgradestack" -STATUS_ACTION = "status" -SETUP_HTTPS_ACTION = "setup-https" -LDAP_SETUP_ACTION = "setup-ldap" -LDAP_SYNC_ACTION = "sync-ldap" -SETUP_GANGLIA_HTTPS_ACTION = "setup-ganglia-https" -ENCRYPT_PASSWORDS_ACTION = "encrypt-passwords" -SETUP_SECURITY_ACTION = "setup-security" -REFRESH_STACK_HASH_ACTION = "refresh-stack-hash" -BACKUP_ACTION = "backup" -RESTORE_ACTION = "restore" - -ACTION_REQUIRE_RESTART = [RESET_ACTION, UPGRADE_ACTION,
[4/8] ambari git commit: AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components - Part 1.4
http://git-wip-us.apache.org/repos/asf/ambari/blob/49955a35/ambari-server/src/main/python/ambari_server/serverSetup.py
--
diff --git a/ambari-server/src/main/python/ambari_server/serverSetup.py
b/ambari-server/src/main/python/ambari_server/serverSetup.py
index 7d45445..b012fc4 100644
--- a/ambari-server/src/main/python/ambari_server/serverSetup.py
+++ b/ambari-server/src/main/python/ambari_server/serverSetup.py
@@ -18,267 +18,396 @@ See the License for the specific language governing
permissions and
limitations under the License.
'''
-import socket
+import optparse
+import os
+import re
+import shutil
import sys
-import urllib2
-from ambari_commons.inet_utils import force_download_file
-from ambari_commons.logging_utils import print_warning_msg, print_error_msg
-from ambari_commons.os_utils import is_root
-from serverConfiguration import *
-from setupSecurity import adjust_directory_permissions, get_is_secure,
store_password_file, encrypt_password, \
+from ambari_commons.exceptions import FatalException, NonFatalException
+from ambari_commons.firewall import Firewall
+from ambari_commons.inet_utils import force_download_file
+from ambari_commons.logging_utils import get_silent, print_info_msg,
print_warning_msg, print_error_msg
+from ambari_commons.os_check import OSConst
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+from ambari_commons.os_utils import run_os_command, is_root
+from ambari_commons.str_utils import compress_backslashes
+from ambari_server.dbConfiguration import DBMSConfigFactory, check_jdbc_drivers
+from ambari_server.serverConfiguration import configDefaults, JDKRelease, \
+ get_ambari_properties, get_full_ambari_classpath, get_java_exe_path,
get_JAVA_HOME, get_value_from_properties, \
+ read_ambari_user, update_properties, validate_jdk, write_property, \
+ JAVA_HOME, JAVA_HOME_PROPERTY, JCE_NAME_PROPERTY, JDBC_RCA_URL_PROPERTY,
JDBC_URL_PROPERTY, \
+ JDK_NAME_PROPERTY, JDK_RELEASES, NR_USER_PROPERTY, OS_FAMILY,
OS_FAMILY_PROPERTY, OS_TYPE, OS_TYPE_PROPERTY, OS_VERSION, \
+ RESOURCES_DIR_PROPERTY, SERVICE_PASSWORD_KEY, SERVICE_USERNAME_KEY,
VIEWS_DIR_PROPERTY, PID_NAME, get_is_secure, \
get_is_persisted
-from userInput import *
-from utils import *
+from ambari_server.setupSecurity import adjust_directory_permissions
+from ambari_server.userInput import get_YN_input, get_validated_string_input
+from ambari_server.utils import locate_file
-if OSCheck.is_windows_family():
- from serverSetup_windows import *
-else:
- # MacOS not supported
- from serverSetup_linux import *
-JDK_INDEX = 0
+# selinux commands
+GET_SE_LINUX_ST_CMD = locate_file('sestatus', '/usr/sbin')
+SE_SETENFORCE_CMD = "setenforce 0"
+SE_STATUS_DISABLED = "disabled"
+SE_STATUS_ENABLED = "enabled"
+SE_MODE_ENFORCING = "enforcing"
+SE_MODE_PERMISSIVE = "permissive"
-def verify_setup_allowed():
- properties = get_ambari_properties()
- if properties == -1:
-print_error_msg("Error getting ambari properties")
-return -1
+# Non-root user setup commands
+NR_USER_COMMENT = "Ambari user"
- isSecure = get_is_secure(properties)
- (isPersisted, masterKeyFile) = get_is_persisted(properties)
- if isSecure and not isPersisted and get_silent():
-print "ERROR: Cannot run silent 'setup' with password encryption enabled "
\
- "and Master Key not persisted."
-print "Ambari Server 'setup' exiting."
-return 1
- return 0
+VIEW_EXTRACT_CMD = "{0} -cp {1}" + \
+ "org.apache.ambari.server.view.ViewRegistry extract {2} " +
\
+ "> " + configDefaults.SERVER_OUT_FILE + " 2>&1"
+MAKE_FILE_EXECUTABLE_CMD = "chmod a+x {0}"
-def check_ambari_user():
- try:
-user = read_ambari_user()
-create_user = False
-update_user_setting = False
-if user is not None:
- create_user = get_YN_input(NR_USER_CHANGE_PROMPT.format(user), False)
- update_user_setting = create_user # Only if we will create another user
-else: # user is not configured yet
- update_user_setting = True # Write configuration anyway
- create_user = get_YN_input(NR_USER_CUSTOMIZE_PROMPT, False)
- if not create_user:
-user = NR_DEFAULT_USER
-
-if create_user:
- (retcode, user) = create_custom_user()
- if retcode != 0:
-return retcode
+# use --no-same-owner when running as root to prevent uucp as the user
(AMBARI-6478)
+UNTAR_JDK_ARCHIVE = "tar --no-same-owner -xvf {0}"
-if update_user_setting:
- write_property(NR_USER_PROPERTY, user)
+JDK_PROMPT = "[{0}] {1}\n"
+JDK_CUSTOM_CHOICE_PROMPT = "[{0}] - Custom
JDK\n==\nEnter
choice ({1}): "
+JDK_VALID_CHOICES = "^[{0}{1:d}]$"
-adjust_directory_permissions(user)
- except OSError as e:
-print_error_msg("Failed: %s" % e.strerror)
-return 4
- except Exception as e:
-print_error
[6/8] ambari git commit: AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components - Part 1.4
http://git-wip-us.apache.org/repos/asf/ambari/blob/49955a35/ambari-server/src/main/python/ambari_server/dbConfiguration.py
--
diff --git a/ambari-server/src/main/python/ambari_server/dbConfiguration.py
b/ambari-server/src/main/python/ambari_server/dbConfiguration.py
index a200879..74bf194 100644
--- a/ambari-server/src/main/python/ambari_server/dbConfiguration.py
+++ b/ambari-server/src/main/python/ambari_server/dbConfiguration.py
@@ -17,12 +17,17 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
+import os
-from ambari_commons import OSCheck
+from ambari_commons import OSConst
from ambari_commons.exceptions import FatalException
-from ambari_commons.logging_utils import print_error_msg
-from ambari_server.setupSecurity import SECURITY_IS_ENCRYPTION_ENABLED
-from serverConfiguration import get_ambari_properties
+from ambari_commons.logging_utils import get_silent, print_error_msg,
print_info_msg, print_warning_msg, set_silent
+from ambari_commons.os_family_impl import OsFamilyImpl
+from ambari_server.serverConfiguration import decrypt_password_for_alias,
get_value_from_properties, get_is_secure, \
+ is_alias_string, \
+ JDBC_PASSWORD_PROPERTY, JDBC_RCA_PASSWORD_ALIAS, PRESS_ENTER_MSG,
get_ambari_properties, update_properties, \
+ RESOURCES_DIR_PROPERTY
+from ambari_server.userInput import get_validated_string_input
#Database settings
@@ -31,77 +36,76 @@ DB_STATUS_RUNNING_DEFAULT = "running"
SETUP_DB_CONNECT_TIMEOUT = 5
SETUP_DB_CONNECT_ATTEMPTS = 3
-DATABASE_INDEX = 0
USERNAME_PATTERN = "^[a-zA-Z_][a-zA-Z0-9_\-]*$"
PASSWORD_PATTERN = "^[a-zA-Z0-9_-]*$"
DATABASE_NAMES = ["postgres", "oracle", "mysql"]
-DATABASE_STORAGE_NAMES = ["Database", "Service", "Database"]
-DATABASE_PORTS = ["5432", "1521", "3306"]
-DATABASE_DRIVER_NAMES = ["org.postgresql.Driver",
"oracle.jdbc.driver.OracleDriver", "com.mysql.jdbc.Driver"]
-DATABASE_CONNECTION_STRINGS = [
- "jdbc:postgresql://{0}:{1}/{2}",
- "jdbc:oracle:thin:@{0}:{1}/{2}",
- "jdbc:mysql://{0}:{1}/{2}"]
-DATABASE_CONNECTION_STRINGS_ALT = [
- "jdbc:postgresql://{0}:{1}/{2}",
- "jdbc:oracle:thin:@{0}:{1}:{2}",
- "jdbc:mysql://{0}:{1}/{2}"]
-ORACLE_SID_PATTERN = "jdbc:oracle:thin:@.+:.+/.+"
-ORACLE_SNAME_PATTERN = "jdbc:oracle:thin:@.+:.+:.+"
-
-DATABASE_CLI_TOOLS = [["psql"], ["sqlplus", "sqlplus64"], ["mysql"]]
-DATABASE_CLI_TOOLS_DESC = ["psql", "sqlplus", "mysql"]
-DATABASE_CLI_TOOLS_USAGE = ['su -postgres --command=psql -f {0} -v
username=\'"{1}"\' -v password="\'{2}\'"',
-'sqlplus {1}/{2} < {0} ',
-'mysql --user={1} --password={2} {3}<{0}']
-
-MYSQL_INIT_SCRIPT =
'/var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql'
-DATABASE_INIT_SCRIPTS =
['/var/lib/ambari-server/resources/Ambari-DDL-Postgres-CREATE.sql',
-
'/var/lib/ambari-server/resources/Ambari-DDL-Oracle-CREATE.sql',
- MYSQL_INIT_SCRIPT]
-DATABASE_DROP_SCRIPTS =
['/var/lib/ambari-server/resources/Ambari-DDL-Postgres-DROP.sql',
-
'/var/lib/ambari-server/resources/Ambari-DDL-Oracle-DROP.sql',
-
'/var/lib/ambari-server/resources/Ambari-DDL-MySQL-DROP.sql']
+DATABASE_FULL_NAMES = {"oracle": "Oracle", "mysql": "MySQL", "postgres":
"PostgreSQL"}
+
+AMBARI_DATABASE_NAME = "ambari"
+AMBARI_DATABASE_TITLE = "ambari"
+
+STORAGE_TYPE_LOCAL = 'local'
+STORAGE_TYPE_REMOTE = 'remote'
+
+DEFAULT_USERNAME = "ambari"
+DEFAULT_PASSWORD = "bigdata"
+
+#
+# Database configuration helper classes
+#
+class DBMSDesc:
+ def __init__(self, i_dbms_key, i_storage_key, i_dbms_name, i_storage_name,
i_fn_create_config):
+self.dbms_key = i_dbms_key
+self.storage_key = i_storage_key
+self.dbms_name = i_dbms_name
+self.storage_name = i_storage_name
+self.fn_create_config = i_fn_create_config
+
+ def create_config(self, options, properties, dbId):
+return self.fn_create_config(options, properties, self.storage_key, dbId)
+
+class DbPropKeys:
+ def __init__(self, i_dbms_key, i_driver_key, i_server_key, i_port_key,
i_db_name_key, i_db_url_key):
+self.dbms_key = i_dbms_key
+self.driver_key = i_driver_key
+self.server_key = i_server_key
+self.port_key = i_port_key
+self.db_name_key = i_db_name_key
+self.db_url_key = i_db_url_key
+
+class DbAuthenticationKeys:
+ def __init__(self, i_user_name_key, i_password_key, i_password_alias,
i_password_filename):
+self.user_name_key = i_user_name_key
+self.password_key = i_password_key
+self.password_alias = i_password_alias
+self.password_filename = i_password_filename
+
#
# Database configuration base class
#
class DBMSConfig(object):
- def __in
[3/8] ambari git commit: AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components - Part 1.4
http://git-wip-us.apache.org/repos/asf/ambari/blob/49955a35/ambari-server/src/main/python/ambari_server/serverSetup_windows.py
--
diff --git a/ambari-server/src/main/python/ambari_server/serverSetup_windows.py
b/ambari-server/src/main/python/ambari_server/serverSetup_windows.py
deleted file mode 100644
index dce404d..000
--- a/ambari-server/src/main/python/ambari_server/serverSetup_windows.py
+++ /dev/null
@@ -1,300 +0,0 @@
-#!/usr/bin/env python
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-import optparse
-import os
-import socket
-import string
-
-from _winreg import (OpenKey, EnumValue, HKEY_LOCAL_MACHINE, KEY_READ,
CloseKey, KEY_WRITE, QueryValueEx, SetValueEx,
- REG_EXPAND_SZ)
-
-from ambari_commons import os_utils
-
-from ambari_commons.exceptions import *
-from ambari_commons.logging_utils import *
-from ambari_commons.os_windows import run_powershell_script, UserHelper,
CHECK_FIREWALL_SCRIPT
-from ambari_server.dbConfiguration import DBMSConfig
-from ambari_server.serverConfiguration import *
-from ambari_server.userInput import get_validated_string_input
-
-# Non-root user setup commands
-NR_USER_COMMENT = "Ambari user"
-NR_GET_OWNER_CMD = 'stat -c "%U" {0}'
-NR_USERADD_CMD = 'cmd /C net user {0} {1} /ADD'
-NR_SET_USER_COMMENT_CMD = 'usermod -c "{0}" {1}'
-
-NR_USER_CHANGE_PROMPT = "Ambari-server service is configured to run under user
'{0}'. Change this setting [y/n] (n)? "
-NR_USER_CUSTOMIZE_PROMPT = "Customize user account for ambari-server service
[y/n] (n)? "
-NR_DEFAULT_USER = "NT AUTHORITY\SYSTEM"
-
-SERVICE_USERNAME_KEY = "TMP_AMBARI_USERNAME"
-SERVICE_PASSWORD_KEY = "TMP_AMBARI_PASSWORD"
-
-# JDK setup choices
-JDK_DEFAULT_CONFIGS = [
- JDKRelease("jdk7.67", "Oracle JDK 7.67",
-
"http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-7u67-windows-x64.exe";,
"jdk-7u67-windows-x64.exe",
-
"http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip";,
"UnlimitedJCEPolicyJDK7.zip",
- None)
-]
-
-JDK_VERSION_REs = ["(jdk.*)/jre", "Creating (jdk.*)/jre"]
-JDK_PROMPT = "[{0}] {1}\n"
-JDK_CUSTOM_CHOICE_PROMPT = "[{0}] - Custom
JDK\n==\nEnter
choice ({1}): "
-JDK_VALID_CHOICES = "^[{0}{1:d}]$"
-CUSTOM_JDK_NUMBER = "4"
-JDK_MIN_FILESIZE = 5000
-MAKE_FILE_EXECUTABLE_CMD = "chmod a+x {0}"
-
-JDK_DOWNLOAD_CMD = "curl --create-dirs -o {0} {1}"
-JDK_DOWNLOAD_SIZE_CMD = "curl -I {0}"
-
-# use --no-same-owner when running as root to prevent uucp as the user
(AMBARI-6478)
-UNTAR_JDK_ARCHIVE = "tar --no-same-owner -xvf {0}"
-
-
-#JDBC
-USERNAME_PATTERN = "^[a-zA-Z_][a-zA-Z0-9_\-]*$"
-DATABASE_DBMS = "sqlserver"
-DATABASE_NAME = "ambari"
-DATABASE_SERVER = "localhostSQLEXPRESS"
-DATABASE_DRIVER_NAME = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
-
-JDBC_PATTERNS = {"sqlserver": "sqljdbc*.jar"}
-DATABASE_FULL_NAMES = {"sqlserver": "SQL Server"}
-JDBC_DB_OPTION_VALUES = ["sqlserver"]
-JDBC_DB_DEFAULT_DRIVER = {"sqlserver" : "sqljdbc4.jar"}
-
-
-ERROR_NOT_ROOT = 'Ambari-server setup should be run with administrator-level
privileges'
-
-MESSAGE_CHECK_FIREWALL = 'Checking firewall status...'
-
-def os_check_firewall():
- out = run_powershell_script(CHECK_FIREWALL_SCRIPT)
- if out[0] != 0:
-print_warning_msg("Unable to check firewall status:{0}".format(out[2]))
-return False
- profiles_status = [i for i in out[1].split("\n") if not i == ""]
- if "1" in profiles_status:
-enabled_profiles = []
-if profiles_status[0] == "1":
- enabled_profiles.append("DomainProfile")
-if profiles_status[1] == "1":
- enabled_profiles.append("StandardProfile")
-if profiles_status[2] == "1":
- enabled_profiles.append("PublicProfile")
-print_warning_msg("Following firewall profiles enabled:{0}. Make sure that
firewall properly configured.".format(",".join(enabled_profiles)))
-return False
- return True
-
-# No security enhancements in Windows
-def disable_security_enhancements():
- retcode = 0
- err = ''
- return (retcode, err)
-
-
-#
-# User account creation
-#
-
-def os_create_custom_user():
- user = get_validated_string_i
ambari git commit: AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components
Repository: ambari
Updated Branches:
refs/heads/trunk b285bd088 -> a902c5372
AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components
Unit test fix for Ubuntu
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a902c537
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a902c537
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a902c537
Branch: refs/heads/trunk
Commit: a902c53720cfbfb8aca5f69a7a7ff579a8911d4b
Parents: b285bd0
Author: Florian Barca
Authored: Mon Jan 19 13:43:40 2015 -0800
Committer: Florian Barca
Committed: Mon Jan 19 13:43:40 2015 -0800
--
ambari-server/src/test/python/TestAmbariServer.py | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/a902c537/ambari-server/src/test/python/TestAmbariServer.py
--
diff --git a/ambari-server/src/test/python/TestAmbariServer.py
b/ambari-server/src/test/python/TestAmbariServer.py
index cb870b8..408c088 100644
--- a/ambari-server/src/test/python/TestAmbariServer.py
+++ b/ambari-server/src/test/python/TestAmbariServer.py
@@ -2305,6 +2305,8 @@
MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
:return: Validates that installation for each database type correctly
stores the database type, database name,
and optionally the postgres schema name.
"""
+from ambari_server import serverConfiguration
+
_ambari_server_.PROMPT_DATABASE_OPTIONS = True
gyni_mock.return_value = True
rp_mock.return_value = "password"
@@ -2337,17 +2339,18 @@
MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
tempdir = tempfile.gettempdir()
os.environ[_ambari_server_.AMBARI_CONF_VAR] = tempdir
+prop_file = os.path.join(tempdir, "ambari.properties")
for i in range(1, 5):
# Use the expected path of the ambari.properties file to delete it if
it exists, and then create a new one
# during each use case.
-prop_file = os.path.join(tempdir, "ambari.properties")
if os.path.exists(prop_file):
os.remove(prop_file)
with open(prop_file, "w") as f:
f.write("server.jdbc.database_name=oldDBName")
f.close()
-_ambari_server_.AMBARI_PROPERTIES_FILE = prop_file
+
+serverConfiguration.AMBARI_PROPERTIES_FILE = prop_file
args = MagicMock()
_ambari_server_.load_default_db_properties(args)
[1/2] ambari git commit: AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components
Repository: ambari
Updated Branches:
refs/heads/trunk 562031851 -> cd2a67c82
http://git-wip-us.apache.org/repos/asf/ambari/blob/cd2a67c8/ambari-server/src/main/python/ambari_server/setupSecurity.py
--
diff --git a/ambari-server/src/main/python/ambari_server/setupSecurity.py
b/ambari-server/src/main/python/ambari_server/setupSecurity.py
index e6879b9..277dd12 100644
--- a/ambari-server/src/main/python/ambari_server/setupSecurity.py
+++ b/ambari-server/src/main/python/ambari_server/setupSecurity.py
@@ -19,18 +19,33 @@ limitations under the License.
'''
import datetime
import fileinput
+import os
import random
+import re
+import shutil
import socket
import stat
+import string
import sys
import tempfile
import urllib2
-from ambari_commons.exceptions import *
-from ambari_commons.os_utils import run_os_command
-from serverConfiguration import *
-from setupActions import *
-from userInput import *
+from ambari_commons.exceptions import NonFatalException, FatalException
+from ambari_commons.logging_utils import get_silent, print_warning_msg,
print_error_msg, print_info_msg
+from ambari_commons.os_check import OSCheck, OSConst
+from ambari_commons.os_utils import copy_file, is_root, is_valid_filepath,
remove_file, set_file_permissions, \
+ run_os_command, search_file
+from ambari_server.serverConfiguration import configDefaults,
get_ambari_properties, read_ambari_user, \
+ get_value_from_properties, find_jdk, get_ambari_classpath, get_conf_dir,
is_alias_string, find_properties_file, \
+ update_properties_2, \
+ JDBC_USE_INTEGRATED_AUTH_PROPERTY, JDBC_PASSWORD_PROPERTY,
JDBC_PASSWORD_FILENAME, \
+ JDBC_RCA_PASSWORD_FILE_PROPERTY, JDBC_RCA_PASSWORD_ALIAS, \
+ JDBC_METRICS_USE_INTEGRATED_AUTH_PROPERTY, JDBC_METRICS_PASSWORD_PROPERTY,
JDBC_METRICS_PASSWORD_FILENAME, \
+ JDBC_METRICS_PASSWORD_ALIAS, \
+ BOOTSTRAP_DIR_PROPERTY, GET_FQDN_SERVICE_URL, BLIND_PASSWORD
+from setupActions import SETUP_ACTION, LDAP_SETUP_ACTION
+from ambari_server.userInput import get_YN_input, get_validated_string_input,
get_validated_filepath_input, \
+ get_prompt_default
SSL_PASSWORD_FILE = "pass.txt"
@@ -59,19 +74,19 @@ SECURITY_PROVIDER_GET_CMD = "{0}" + os.sep + "bin" + os.sep
+ java_bin + " -cp {
os.pathsep + "{2} " +\
"org.apache.ambari.server.security.encryption" +\
".CredentialProvider GET {3} {4} {5} " +\
- "> " + SERVER_OUT_FILE + " 2>&1"
+ "> " + configDefaults.SERVER_OUT_FILE + " 2>&1"
SECURITY_PROVIDER_PUT_CMD = "{0}" + os.sep + "bin" + os.sep + java_bin + " -cp
{1}" +\
os.pathsep + "{2} " +\
"org.apache.ambari.server.security.encryption" +\
".CredentialProvider PUT {3} {4} {5} " +\
- "> " + SERVER_OUT_FILE + " 2>&1"
+ "> " + configDefaults.SERVER_OUT_FILE + " 2>&1"
SECURITY_PROVIDER_KEY_CMD = "{0}" + os.sep + "bin" + os.sep + java_bin + " -cp
{1}" +\
os.pathsep + "{2} " +\
"org.apache.ambari.server.security.encryption" +\
".MasterKeyServiceImpl {3} {4} {5} " +\
- "> " + SERVER_OUT_FILE + " 2>&1"
+ "> " + configDefaults.SERVER_OUT_FILE + " 2>&1"
SSL_KEY_DIR = 'security.server.keys_dir'
SSL_API_PORT = 'client.api.ssl.port'
@@ -128,32 +143,6 @@ REGEX_ANYTHING = ".*"
CLIENT_SECURITY_KEY = "client.security"
-# ownership/permissions mapping
-# path - permissions - user - group - recursive
-# Rules are executed in the same order as they are listed
-# {0} in user/group will be replaced by customized ambari-server username
-NR_ADJUST_OWNERSHIP_LIST = [
-
- ("/var/log/ambari-server", "644", "{0}", True),
- ("/var/log/ambari-server", "755", "{0}", False),
- ("/var/run/ambari-server", "644", "{0}", True),
- ("/var/run/ambari-server", "755", "{0}", False),
- ("/var/run/ambari-server/bootstrap", "755", "{0}", False),
- ("/var/lib/ambari-server/ambari-env.sh", "700", "{0}", False),
- ("/var/lib/ambari-server/keys", "600", "{0}", True),
- ("/var/lib/ambari-server/keys", "700", "{0}", False),
- ("/var/lib/ambari-server/keys/db", "700", "{0}", False),
- ("/var/lib/ambari-server/keys/db/newcerts", "700", "{0}", False),
- ("/var/lib/ambari-server/keys/.ssh", "700", "{0}", False),
- ("/var/lib/ambari-server/resources/stacks/", "755", "{0}", True),
- ("/var/lib/ambari-server/resources/custom_actions/", "755", "{0}", True),
- ("/etc/ambari-server/conf", "644", "{0}", True),
- ("/etc/ambari-server/conf", "755", "{0}", False),
- ("/etc/ambari-server/conf/password.dat", "640", "{0}", False),
- # Also, /etc/ambari-server/conf/password.dat
- # is generated later at store_password_file
-]
-
def is_valid_https_port(port):
prope
[2/2] ambari git commit: AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components
AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components +Merged serviceConfiguration*.py into serviceConfiguration.py +Moved a series of configuration settings away from ambari-server.py +Further optimized imports Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cd2a67c8 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cd2a67c8 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cd2a67c8 Branch: refs/heads/trunk Commit: cd2a67c8277bb899bb886f00f044a95fbf88fc06 Parents: 5620318 Author: Florian Barca Authored: Wed Jan 14 23:04:04 2015 -0800 Committer: Florian Barca Committed: Wed Jan 14 23:04:04 2015 -0800 -- .../src/main/python/ambari_agent/HostCleanup.py | 8 - .../src/main/python/ambari-server-windows.py| 68 +- ambari-server/src/main/python/ambari-server.py | 684 +++ .../ambari_server/dbConfiguration_windows.py| 34 +- .../src/main/python/ambari_server/properties.py | 15 +- .../python/ambari_server/serverConfiguration.py | 291 +--- .../ambari_server/serverConfiguration_linux.py | 67 -- .../serverConfiguration_windows.py | 98 --- .../main/python/ambari_server/serverSetup.py| 85 ++- .../python/ambari_server/serverSetup_linux.py | 15 +- .../python/ambari_server/serverSetup_windows.py | 4 +- .../main/python/ambari_server/setupSecurity.py | 82 +-- .../src/main/python/ambari_server/userInput.py | 6 + .../src/test/python/TestAmbariServer.py | 542 +++ ambari-server/src/test/python/TestOSCheck.py| 20 +- 15 files changed, 750 insertions(+), 1269 deletions(-) -- http://git-wip-us.apache.org/repos/asf/ambari/blob/cd2a67c8/ambari-agent/src/main/python/ambari_agent/HostCleanup.py -- diff --git a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py index a128dad..7aeb70a 100644 --- a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py +++ b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py @@ -475,14 +475,6 @@ class HostCleanup: (stdoutdata, stderrdata) = process.communicate() return process.returncode, stdoutdata, stderrdata - - def search_file(self, filename, search_path, pathsep=os.pathsep): -""" Given a search path, find file with requested name """ -for path in string.split(search_path, pathsep): - candidate = os.path.join(path, filename) - if os.path.exists(candidate): return os.path.abspath(candidate) -return None - # Copy file and save with file.# (timestamp) def backup_file(filePath): if filePath is not None and os.path.exists(filePath): http://git-wip-us.apache.org/repos/asf/ambari/blob/cd2a67c8/ambari-server/src/main/python/ambari-server-windows.py -- diff --git a/ambari-server/src/main/python/ambari-server-windows.py b/ambari-server/src/main/python/ambari-server-windows.py index 5f9d58f..acd91ab 100644 --- a/ambari-server/src/main/python/ambari-server-windows.py +++ b/ambari-server/src/main/python/ambari-server-windows.py @@ -19,22 +19,32 @@ limitations under the License. ''' import optparse +import os +import sys import subprocess -from ambari_commons.ambari_service import AmbariService, ENV_PYTHON_PATH -from ambari_commons.logging_utils import get_verbose, set_verbose, get_silent, set_silent, get_debug_mode, set_debug_mode +from ambari_commons.ambari_service import AmbariService +from ambari_commons.exceptions import FatalException, NonFatalException +from ambari_commons.logging_utils import print_info_msg, print_warning_msg, print_error_msg, \ + get_verbose, set_verbose, get_silent, set_silent, get_debug_mode, set_debug_mode +from ambari_commons.os_utils import remove_file, set_open_files_limit from ambari_commons.os_windows import SvcStatusCallback from ambari_server import utils from ambari_server.dbConfiguration import DBMSConfig from ambari_server.resourceFilesKeeper import ResourceFilesKeeper, KeeperException -from ambari_server.serverConfiguration import * +from ambari_server.serverConfiguration import find_jdk, get_ambari_classpath, get_ambari_properties, get_conf_dir, \ + get_value_from_properties, configDefaults, DEBUG_MODE_KEY, RESOURCES_DIR_DEFAULT, RESOURCES_DIR_PROPERTY, \ + SERVER_OUT_FILE_KEY, STACK_LOCATION_DEFAULT, STACK_LOCATION_KEY, SUSPEND_START_MODE_KEY, VERBOSE_OUTPUT_KEY from ambari_server.serverSetup import setup, reset, is_server_running, upgrade -from ambari_server.setupActions import * -from ambari_server.setupSecurity import * from ambari_server.serverSetup_windows import SERVICE_PASSWORD_KEY, SERVICE_USERNAME_KEY +from ambari_server.setupActions import SETUP_A
[3/3] ambari git commit: AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components
AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components
+Fixed ambari_server import in TestAmbariServer.py, to allow future imports
from modules in the ambari_server subdir
+Moved SILENT, VERBOSE and DEBUG_MODE to logging_utils.py, to allow future use
from modular code
+Moved run_os_command to os_utils
+Extended the set of test platform primitives
+Optimized imports
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/16bee385
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/16bee385
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/16bee385
Branch: refs/heads/trunk
Commit: 16bee3858711178dd4025b56211f03b11fa75954
Parents: fb947b0
Author: Florian Barca
Authored: Tue Jan 13 13:46:04 2015 -0800
Committer: Florian Barca
Committed: Tue Jan 13 13:46:04 2015 -0800
--
.../test/python/ambari_agent/TestHostInfo.py| 40 +-
.../python/ambari_agent/TestRegistration.py |6 +-
.../src/main/python/ambari_commons/firewall.py | 21 +-
.../main/python/ambari_commons/logging_utils.py | 32 +-
.../src/main/python/ambari_commons/os_linux.py | 14 +-
.../src/main/python/ambari_commons/os_utils.py | 21 +-
.../main/python/ambari_commons/os_windows.py| 168 +-
.../core/providers/windows/system.py|4 +-
.../functions/get_unique_id_and_date.py |2 +-
.../src/test/python/only_for_platform.py| 12 +
ambari-server/conf/windows/ambari.properties|1 +
.../src/main/python/ambari-server-windows.py| 22 +-
ambari-server/src/main/python/ambari-server.py | 57 +-
.../ambari_server/dbConfiguration_linux.py |8 +-
.../python/ambari_server/serverConfiguration.py |8 +
.../main/python/ambari_server/serverSetup.py|5 +-
.../python/ambari_server/serverSetup_linux.py |5 +-
.../python/ambari_server/serverSetup_windows.py |2 +-
.../main/python/ambari_server/setupSecurity.py |6 +-
.../src/main/python/ambari_server/userInput.py | 12 +-
.../custom_actions/scripts/install_packages.py |8 +-
.../src/test/python/TestAmbariServer.py | 2614 +-
.../test/python/custom_actions/TestCheckHost.py | 18 +-
23 files changed, 1574 insertions(+), 1512 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
--
diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
index 338d080..adeb798 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
@@ -31,7 +31,7 @@ from mock.mock import create_autospec
import ambari_commons
from ambari_commons import OSCheck
import os
-from only_for_platform import only_for_platform, get_platform,
PLATFORM_WINDOWS, PLATFORM_LINUX
+from only_for_platform import not_for_platform, get_platform,
PLATFORM_WINDOWS, PLATFORM_LINUX
if get_platform() != PLATFORM_WINDOWS:
os_distro_value = ('Suse','11','Final')
@@ -51,7 +51,7 @@ with patch.object(OSCheck, "os_distribution", new =
MagicMock(return_value = os_
@patch.object(OSCheck, "os_distribution", new = MagicMock(return_value =
os_distro_value))
class TestHostInfo(TestCase):
- @only_for_platform(PLATFORM_LINUX)
+ @not_for_platform(PLATFORM_WINDOWS)
@patch.object(OSCheck, 'get_os_family')
@patch('resource_management.libraries.functions.packages_analyzer.subprocessWithTimeout')
def test_analyze_zypper_out(self, spwt_mock, get_os_family_mock):
@@ -82,7 +82,7 @@ class TestHostInfo(TestCase):
self.assertTrue(installedPackages[3][2], "HDP")
self.assertTrue(installedPackages[6][1], "11-38.13.9")
- @only_for_platform(PLATFORM_LINUX)
+ @not_for_platform(PLATFORM_WINDOWS)
def test_getReposToRemove(self):
l1 = ["Hortonworks Data Platform Utils Version - HDP-UTILS-1.1.0.15",
"Ambari 1.x", "HDP"]
l2 = ["Ambari", "HDP-UTIL"]
@@ -96,7 +96,7 @@ class TestHostInfo(TestCase):
self.assertTrue(1, len(l3))
self.assertEqual(l3[0], "HDP-1.3.0")
- @only_for_platform(PLATFORM_LINUX)
+ @not_for_platform(PLATFORM_WINDOWS)
def test_perform_package_analysis(self):
installedPackages = [
["hadoop-a", "2.3", "HDP"], ["zk", "3.1", "HDP"], ["webhcat", "3.1",
"HDP"],
@@ -143,7 +143,7 @@ class TestHostInfo(TestCase):
for package in expected:
self.assertTrue(package in allPackages)
- @only_for_platform(PLATFORM_LINUX)
+ @not_for_platform(PLATFORM_WINDOWS)
@patch.object(OSCheck, 'get_os_family')
@patch('resource_management.libraries.functions.packages_analyzer.subprocessWithTimeout')
def test_analyze_yum_output(self, subprocessWith
[1/3] ambari git commit: AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components
Repository: ambari
Updated Branches:
refs/heads/trunk fb947b0cc -> 16bee3858
http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-server/src/test/python/custom_actions/TestCheckHost.py
--
diff --git a/ambari-server/src/test/python/custom_actions/TestCheckHost.py
b/ambari-server/src/test/python/custom_actions/TestCheckHost.py
index f910d06..cb4c540 100644
--- a/ambari-server/src/test/python/custom_actions/TestCheckHost.py
+++ b/ambari-server/src/test/python/custom_actions/TestCheckHost.py
@@ -23,7 +23,7 @@ import json
import os
import socket
import subprocess
-from ambari_commons import inet_utils
+from ambari_commons import inet_utils, OSCheck
from resource_management import Script, ConfigDictionary
from mock.mock import patch
from mock.mock import MagicMock
@@ -31,6 +31,13 @@ from unittest import TestCase
from check_host import CheckHost
+from only_for_platform import only_for_platform, get_platform, PLATFORM_LINUX,
PLATFORM_WINDOWS
+
+if get_platform() != PLATFORM_WINDOWS:
+ os_distro_value = ('Suse','11','Final')
+else:
+ os_distro_value = ('win2012serverr2','6.3','WindowsServer')
+
class TestCheckHost(TestCase):
@@ -232,7 +239,7 @@ class TestCheckHost(TestCase):
structured_out_mock.assert_called_with({})
- @patch("platform.linux_distribution")
+ @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value =
os_distro_value))
@patch("platform.system")
@patch.object(Script, 'get_config')
@patch.object(Script, 'get_tmp_dir')
@@ -246,12 +253,7 @@ class TestCheckHost(TestCase):
@patch('time.time')
def testLastAgentEnv(self, time_mock, checkReverseLookup_mock,
checkIptables_mock, getTransparentHugePage_mock,
getUMask_mock, checkLiveServices_mock, javaProcs_mock,
put_structured_out_mock,
- get_tmp_dir_mock, get_config_mock, systemMock,
distMock):
-
-
-systemMock.return_value = "Linux"
-distMock.return_value = ("CentOS", "6.3", None)
-
+ get_tmp_dir_mock, get_config_mock, systemmock):
jsonFilePath = os.path.join("../resources/custom_actions",
"check_last_agent_env.json")
with open(jsonFilePath, "r") as jsonFile:
jsonPayload = json.load(jsonFile)
[2/3] ambari git commit: AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components
http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-server/src/test/python/TestAmbariServer.py
--
diff --git a/ambari-server/src/test/python/TestAmbariServer.py
b/ambari-server/src/test/python/TestAmbariServer.py
index e7014fc..71d57a0 100644
--- a/ambari-server/src/test/python/TestAmbariServer.py
+++ b/ambari-server/src/test/python/TestAmbariServer.py
@@ -15,38 +15,48 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
-
from stacks.utils.RMFTestCase import *
-import StringIO
-import re
-from unittest import TestCase
import sys
-from mock.mock import patch, MagicMock, create_autospec
-import os, errno, tempfile
-import signal
-import stat
+import os
import datetime
-import operator
+import errno
import json
+from mock.mock import patch, MagicMock, create_autospec
+import operator
from optparse import OptionParser
import platform
+import re
import shutil
-from pwd import getpwnam
+import signal
+import stat
+import StringIO
+import tempfile
+from unittest import TestCase
+
+from only_for_platform import only_for_platform, get_platform, PLATFORM_LINUX,
PLATFORM_WINDOWS
+
+from ambari_commons import Firewall, OSCheck, OSConst, FirewallChecks
+from ambari_commons.logging_utils import get_verbose, set_verbose, get_silent,
set_silent, get_debug_mode
from ambari_server.resourceFilesKeeper import ResourceFilesKeeper,
KeeperException
from ambari_server import BackupRestore
-
-# We have to use this import HACK because the filename contains a dash
-from ambari_commons import Firewall, OSCheck, OSConst, FirewallChecks
-with patch("platform.linux_distribution", return_value =
('Suse','11','Final')):
+if get_platform() != PLATFORM_WINDOWS:
+ os_distro_value = ('Suse','11','Final')
+ from pwd import getpwnam
+else:
+ #No Windows tests for now, but start getting prepared
+ os_distro_value = ('win2012serverr2','6.3','WindowsServer')
+
+# We have to use this import HACK because the filename contains a dash
+with patch("platform.linux_distribution", return_value = os_distro_value):
with patch("os.symlink"):
with patch("__builtin__.open"):
with patch("glob.glob", return_value = ['/etc/init.d/postgresql-9.3']):
-ambari_server = __import__('ambari-server')
+_ambari_server_ = __import__('ambari-server')
-FatalException = ambari_server.FatalException
-NonFatalException = ambari_server.NonFatalException
+FatalException = _ambari_server_.FatalException
+NonFatalException = _ambari_server_.NonFatalException
CURR_AMBARI_VERSION = "2.0.0"
class TestAmbariServer(TestCase):
@@ -59,8 +69,8 @@ class TestAmbariServer(TestCase):
sys.stdout = sys.__stdout__
- @patch.object(ambari_server, 'configure_database_username_password')
- @patch.object(ambari_server, 'run_os_command')
+ @patch.object(_ambari_server_, 'configure_database_username_password')
+ @patch.object(_ambari_server_, 'run_os_command')
@patch('optparse.Values')
def test_configure_pg_hba_ambaridb_users(self, OptParseValuesMock,
run_os_command_method,
@@ -70,15 +80,15 @@ class TestAmbariServer(TestCase):
opvm = OptParseValuesMock.return_value
opvm.database_username = "ffdf"
tf1 = tempfile.NamedTemporaryFile()
-ambari_server.PG_HBA_CONF_FILE = tf1.name
+_ambari_server_.PG_HBA_CONF_FILE = tf1.name
# Run test
-ambari_server.configure_pg_hba_ambaridb_users()
+_ambari_server_.configure_pg_hba_ambaridb_users()
# Check results
self.assertTrue(run_os_command_method.called)
self.assertTrue(configure_database_username_password_method.called)
string_expected = self.get_file_string(self
.get_samples_dir("configure_pg_hba_ambaridb_users1"))
-string_actual = self.get_file_string(ambari_server.PG_HBA_CONF_FILE)
+string_actual = self.get_file_string(_ambari_server_.PG_HBA_CONF_FILE)
self.assertEquals(string_expected, string_actual)
pass
@@ -86,9 +96,9 @@ class TestAmbariServer(TestCase):
def test_servicename_regex(self, raw_input_method):
''' Test to make sure the service name can contain digits '''
-ambari_server.SILENT = False
+set_silent(False)
raw_input_method.return_value = "OT100"
-result = ambari_server.get_validated_service_name("ambari", 1)
+result = _ambari_server_.get_validated_service_name("ambari", 1)
self.assertEqual("OT100", result, "Not accepting digits")
pass
@@ -96,31 +106,31 @@ class TestAmbariServer(TestCase):
def test_dbname_regex(self, raw_input_method):
''' Test to make sure the service name can contain digits '''
-ambari_server.SILENT = False
+set_silent(False)
raw_input_method.return_value = "OT100"
-result = ambari_server.get_validated_db_name("ambari")
+result = _ambari_server_.get_valida
ambari git commit: AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components
Repository: ambari
Updated Branches:
refs/heads/trunk 1767fcfa4 -> e4b2d34b6
AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components
Pass 1.1
Preliminary changes. Introducing OS-dependent abstraction for functions.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e4b2d34b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e4b2d34b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e4b2d34b
Branch: refs/heads/trunk
Commit: e4b2d34b6bf1f1c31944576277cb71f8d771c2e8
Parents: 1767fcf
Author: Florian Barca
Authored: Mon Jan 5 15:51:48 2015 -0800
Committer: Florian Barca
Committed: Mon Jan 5 15:51:48 2015 -0800
--
.../python/ambari_commons/os_family_impl.py | 42 +---
1 file changed, 37 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/e4b2d34b/ambari-common/src/main/python/ambari_commons/os_family_impl.py
--
diff --git a/ambari-common/src/main/python/ambari_commons/os_family_impl.py
b/ambari-common/src/main/python/ambari_commons/os_family_impl.py
index f8a3379..65a502a 100644
--- a/ambari-common/src/main/python/ambari_commons/os_family_impl.py
+++ b/ambari-common/src/main/python/ambari_commons/os_family_impl.py
@@ -17,17 +17,20 @@ limitations under the License.
'''
import types
-from os_check import OSCheck
+from ambari_commons import OSCheck
class OsFamilyImpl(object):
"""
- Base class for os depended factory. Usage::
+ Base class for os dependent factory. Usage::
class BaseFoo(object): pass
- @Factory("windows")
- class OsFoo(object):pass
- print BaseFoo()# OsFoo
+ @OsFamilyImpl(os_family="windows")
+ class OsFooW(BaseFoo):pass
+ print BaseFoo()# OsFooW
+ @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+ class OsFooD(BaseFoo):pass
+ print BaseFoo()# OsFooD
"""
@@ -62,3 +65,32 @@ class OsFamilyImpl(object):
base_cls.__new__ = types.MethodType(new, base_cls)
return cls
+
+class OsFamilyFuncImpl(object):
+ """
+ Base class for os dependent function. Usage::
+
+ @OSFamilyFuncImpl(os_family="windows")
+ def os_foo(...):pass
+
+ """
+ _func_impls = {}
+
+ def _createFunctionInstance(self, func):
+self._func_impls[func.__module__ + "." + func.__name__ + "." +
self.os_const] = func
+
+def thunk(*args, **kwargs):
+ fn_id_base = func.__module__ + "." + func.__name__
+ fn_id = fn_id_base + "." + OSCheck.get_os_family()
+ if fn_id not in self._func_impls:
+fn_id = fn_id_base + "." + OsFamilyImpl.DEFAULT
+
+ fn = self._func_impls[fn_id]
+ return fn(*args, **kwargs)
+return thunk
+
+ def __init__(self, os_family):
+self.os_const = os_family
+
+ def __call__(self, func):
+return self._createFunctionInstance(func)
