AMBARI-15114: Parameterize distro-specific stack information for HBASE (Juanjo Marron via dili)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fa8b4c4e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fa8b4c4e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fa8b4c4e Branch: refs/heads/AMBARI-13364 Commit: fa8b4c4e3981c82bf9930526f3bee8581da0524b Parents: 6900539 Author: Di Li <d...@apache.org> Authored: Mon Feb 29 10:32:35 2016 -0500 Committer: Jayush Luniya <jlun...@hortonworks.com> Committed: Wed Mar 9 15:36:28 2016 -0800 ---------------------------------------------------------------------- .../0.96.0.2.0/package/scripts/hbase_client.py | 5 +-- .../0.96.0.2.0/package/scripts/hbase_master.py | 3 +- .../package/scripts/hbase_regionserver.py | 3 +- .../0.96.0.2.0/package/scripts/params_linux.py | 33 +++++++++++--------- .../package/scripts/phoenix_queryserver.py | 7 +++-- .../package/scripts/phoenix_service.py | 2 +- .../0.96.0.2.0/package/scripts/status_params.py | 10 ++++-- .../HBASE/0.96.0.2.0/package/scripts/upgrade.py | 2 +- .../HDP/2.0.6/configuration/cluster-env.xml | 10 ++++++ 9 files changed, 48 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/fa8b4c4e/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_client.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_client.py index c31bbf6..2f59e4b 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_client.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_client.py @@ -51,13 +51,14 @@ class HbaseClientWindows(HbaseClient): @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) class HbaseClientDefault(HbaseClient): def get_stack_to_component(self): - return {"HDP": "hbase-client"} + import params + return {params.stack_name: "hbase-client"} def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) - if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0: + if params.version and compare_versions(format_stack_version(params.version), params.stack_version_ru_support) >= 0: conf_select.select(params.stack_name, "hbase", params.version) stack_select.select("hbase-client", params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/fa8b4c4e/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py index 9515f61..5833773 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py @@ -71,7 +71,8 @@ class HbaseMasterWindows(HbaseMaster): @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) class HbaseMasterDefault(HbaseMaster): def get_stack_to_component(self): - return {"HDP": "hbase-master"} + import params + return {params.stack_name: "hbase-master"} def pre_upgrade_restart(self, env, upgrade_type=None): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/fa8b4c4e/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py index c17d219..37dd0a1 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py @@ -69,7 +69,8 @@ class HbaseRegionServerWindows(HbaseRegionServer): @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) class HbaseRegionServerDefault(HbaseRegionServer): def get_stack_to_component(self): - return {"HDP": "hbase-regionserver"} + import params + return {params.stack_name: "hbase-regionserver"} def pre_upgrade_restart(self, env, upgrade_type=None): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/fa8b4c4e/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py index 63e548a..6a2abab 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py @@ -43,15 +43,18 @@ exec_tmp_dir = Script.get_tmp_dir() sudo = AMBARI_SUDO_BINARY stack_name = default("/hostLevelParams/stack_name", None) +stack_dir = config['configurations']['cluster-env']['stack_dir'] agent_stack_retry_on_unavailability = cbool(config["hostLevelParams"]["agent_stack_retry_on_unavailability"]) agent_stack_retry_count = cint(config["hostLevelParams"]["agent_stack_retry_count"]) - version = default("/commandParams/version", None) component_directory = status_params.component_directory etc_prefix_dir = "/etc/hbase" stack_version_unformatted = str(config['hostLevelParams']['stack_version']) stack_version_formatted = format_stack_version(stack_version_unformatted) +stack_version_ru_support = config['configurations']['cluster-env']['stack_version_ru_support'] +stack_version_ranger_support = config['configurations']['cluster-env']['stack_version_ranger_support'] +stack_version_phoenix_support = config['configurations']['cluster-env']['stack_version_phoenix_support'] # hadoop default parameters hadoop_bin_dir = stack_select.get_hadoop_dir("bin") @@ -62,19 +65,19 @@ region_drainer = "/usr/lib/hbase/bin/draining_servers.rb" hbase_cmd = "/usr/lib/hbase/bin/hbase" hbase_max_direct_memory_size = None -# hadoop parameters for 2.2+ -if Script.is_stack_greater_or_equal("2.2"): - daemon_script = format('/usr/hdp/current/hbase-client/bin/hbase-daemon.sh') - region_mover = format('/usr/hdp/current/hbase-client/bin/region_mover.rb') - region_drainer = format('/usr/hdp/current/hbase-client/bin/draining_servers.rb') - hbase_cmd = format('/usr/hdp/current/hbase-client/bin/hbase') +# hadoop parameters for stack_version_ru_support+ +if Script.is_stack_greater_or_equal(stack_version_ru_support): + daemon_script = format('{stack_dir}/current/hbase-client/bin/hbase-daemon.sh') + region_mover = format('{stack_dir}/current/hbase-client/bin/region_mover.rb') + region_drainer = format('{stack_dir}/current/hbase-client/bin/draining_servers.rb') + hbase_cmd = format('{stack_dir}/current/hbase-client/bin/hbase') hbase_max_direct_memory_size = default('configurations/hbase-env/hbase_max_direct_memory_size', None) - daemon_script=format("/usr/hdp/current/{component_directory}/bin/hbase-daemon.sh") - region_mover = format("/usr/hdp/current/{component_directory}/bin/region_mover.rb") - region_drainer = format("/usr/hdp/current/{component_directory}/bin/draining_servers.rb") - hbase_cmd = format("/usr/hdp/current/{component_directory}/bin/hbase") + daemon_script=format("{stack_dir}/current/{component_directory}/bin/hbase-daemon.sh") + region_mover = format("{stack_dir}/current/{component_directory}/bin/region_mover.rb") + region_drainer = format("{stack_dir}/current/{component_directory}/bin/draining_servers.rb") + hbase_cmd = format("{stack_dir}/current/{component_directory}/bin/hbase") hbase_conf_dir = status_params.hbase_conf_dir @@ -84,7 +87,7 @@ hbase_user_nofile_limit = default("/configurations/hbase-env/hbase_user_nofile_l hbase_user_nproc_limit = default("/configurations/hbase-env/hbase_user_nproc_limit", "16000") # no symlink for phoenix-server at this point -phx_daemon_script = '/usr/hdp/current/phoenix-server/bin/queryserver.py' +phx_daemon_script = '{stack_dir}/current/phoenix-server/bin/queryserver.py' hbase_excluded_hosts = config['commandParams']['excluded_hosts'] hbase_drain_only = default("/commandParams/mark_draining_only",False) @@ -311,7 +314,7 @@ if has_ranger_admin: downloaded_custom_connector = format("{exec_tmp_dir}/{jdbc_jar_name}") driver_curl_source = format("{jdk_location}/{jdbc_symlink_name}") - driver_curl_target = format("/usr/hdp/current/{component_directory}/lib/{jdbc_jar_name}") + driver_curl_target = format("{stack_dir}/current/{component_directory}/lib/{jdbc_jar_name}") hbase_ranger_plugin_config = { 'username': repo_config_username, @@ -351,11 +354,11 @@ if security_enabled: hbase_coprocessor_master_classes = "org.apache.hadoop.hbase.security.access.AccessController" hbase_coprocessor_regionserver_classes = "org.apache.hadoop.hbase.security.access.AccessController" hbase_coprocessor_region_classes = "org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,org.apache.hadoop.hbase.security.access.AccessController" - elif xml_configurations_supported: # HDP stack 2.3+ ranger plugin enabled + elif xml_configurations_supported: # stack_version_ranger_support+ hbase_coprocessor_master_classes = "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor " hbase_coprocessor_regionserver_classes = "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor" hbase_coprocessor_region_classes = "org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor" - else: # HDP Stack 2.2 and less / ranger plugin enabled + else: # stack_version_ru_support and less / ranger plugin enabled hbase_coprocessor_master_classes = "com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor" hbase_coprocessor_regionserver_classes = "com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor" hbase_coprocessor_region_classes = "org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor" http://git-wip-us.apache.org/repos/asf/ambari/blob/fa8b4c4e/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_queryserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_queryserver.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_queryserver.py index 87e4899..c28fa7d 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_queryserver.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_queryserver.py @@ -23,7 +23,7 @@ from resource_management.libraries.script import Script from phoenix_service import phoenix_service from hbase import hbase -# Note: Phoenix Query Server is only applicable to HDP-2.3 and above. +# Note: Phoenix Query Server is only applicable to stack_version_phoenix_support and above. class PhoenixQueryServer(Script): def install(self, env): @@ -33,7 +33,8 @@ class PhoenixQueryServer(Script): def get_stack_to_component(self): - return {"HDP": "phoenix-server"} + import params + return {params.stack_name: "phoenix-server"} def configure(self, env): @@ -59,7 +60,7 @@ class PhoenixQueryServer(Script): import params env.set_params(params) - if Script.is_stack_greater_or_equal("2.3"): + if Script.is_stack_greater_or_equal(params.stack_version_phoenix_support): # phoenix uses hbase configs conf_select.select(params.stack_name, "hbase", params.version) stack_select.select("phoenix-server", params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/fa8b4c4e/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_service.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_service.py index e155cec..33c880c 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_service.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_service.py @@ -22,7 +22,7 @@ from resource_management.core.resources.system import Execute from resource_management.core.resources.system import File from resource_management.libraries.functions import check_process_status, format -# Note: Phoenix Query Server is only applicable to HDP-2.3 and above. +# Note: Phoenix Query Server is only applicable to stack_version_phoenix_support and above. def phoenix_service(action = 'start'): # 'start', 'stop', 'status' # Note: params/status_params should already be imported before calling phoenix_service() pid_file = format("{pid_dir}/phoenix-{hbase_user}-server.pid") http://git-wip-us.apache.org/repos/asf/ambari/blob/fa8b4c4e/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py index 535c821..887f50f 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py @@ -25,7 +25,7 @@ from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.script.script import Script # a map of the Ambari role to the component name -# for use with /usr/hdp/current/<component> +# for use with <stack_dir>/current/<component> SERVER_ROLE_DIRECTORY_MAP = { 'HBASE_MASTER' : 'hbase-master', 'HBASE_REGIONSERVER' : 'hbase-regionserver', @@ -49,7 +49,11 @@ else: kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) tmp_dir = Script.get_tmp_dir() + # Stack related params + stack_version_ru_support = config['configurations']['cluster-env']['stack_version_ru_support'] + stack_dir = config['configurations']['cluster-env']['stack_dir'] + hbase_conf_dir = "/etc/hbase/conf" limits_conf_dir = "/etc/security/limits.d" - if Script.is_stack_greater_or_equal("2.2"): - hbase_conf_dir = format("/usr/hdp/current/{component_directory}/conf") \ No newline at end of file + if Script.is_stack_greater_or_equal(stack_version_ru_support): + hbase_conf_dir = format("{stack_dir}/current/{component_directory}/conf") http://git-wip-us.apache.org/repos/asf/ambari/blob/fa8b4c4e/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/upgrade.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/upgrade.py index 92c0f70..467139f 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/upgrade.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/upgrade.py @@ -29,7 +29,7 @@ from resource_management.libraries.functions.decorator import retry def prestart(env, stack_component): import params - if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0: + if params.version and compare_versions(format_stack_version(params.version), params.stack_version_ru_support) >= 0: conf_select.select(params.stack_name, "hbase", params.version) stack_select.select(stack_component, params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/fa8b4c4e/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml index 3bef518..75b499f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml @@ -57,6 +57,16 @@ <description>Stack version from which hadoop-hdfs-nfs3 is supported</description> </property> <property> + <name>stack_version_ranger_support</name> + <value>2.3.0.0</value> + <description>Stack version from which ranger is supported</description> + </property> + <property> + <name>stack_version_phoenix_support</name> + <value>2.3.0.0</value> + <description>Stack version from which Phoenix Query Server is supported</description> + </property> + <property> <name>stack_version_oozie_preconfiguration_support</name> <value>2.3.0.0</value> <description>Stack version from which oozie-server special case in the PRE-UPGRADE phase is supported</description>
