Repository: ambari Updated Branches: refs/heads/trunk 0dc911e3a -> 2209b7203
AMBARI-11667. Start NN in HA, Mapreduce, Oozie and some other are failing with HTTP SPNEGO configured.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2209b720 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2209b720 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2209b720 Branch: refs/heads/trunk Commit: 2209b7203d92e25efd0b12fd1eb1bde61a039c72 Parents: 0dc911e Author: Vitaly Brodetskyi <vbrodets...@hortonworks.com> Authored: Thu Jun 4 05:06:27 2015 +0300 Committer: Vitaly Brodetskyi <vbrodets...@hortonworks.com> Committed: Thu Jun 4 05:06:27 2015 +0300 ---------------------------------------------------------------------- .../libraries/functions/jmx.py | 19 ++++++++++++++++--- .../libraries/functions/namenode_ha_utils.py | 14 +++++++------- .../libraries/providers/hdfs_resource.py | 6 ++++-- .../package/scripts/namenode_ha_state.py | 4 ++-- .../2.1.0.2.0/package/scripts/params_linux.py | 4 ++++ .../python/stacks/2.0.6/HDFS/test_journalnode.py | 6 ++++-- 6 files changed, 37 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/2209b720/ambari-common/src/main/python/resource_management/libraries/functions/jmx.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/jmx.py b/ambari-common/src/main/python/resource_management/libraries/functions/jmx.py index a3cfb1b..be346e5 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/jmx.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/jmx.py @@ -19,13 +19,26 @@ limitations under the License. ''' import urllib2 import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. +from resource_management.core import shell +from resource_management.core.logger import Logger -def get_value_from_jmx(qry, property): +def get_value_from_jmx(qry, property, security_enabled, run_user, is_https_enabled): try: - response = urllib2.urlopen(qry) - data = response.read() + if security_enabled: + cmd = ['curl', '--negotiate', '-u', ':'] + else: + cmd = ['curl'] + + if is_https_enabled: + cmd.append("-k") + + cmd.append(qry) + + _, data = shell.checked_call(cmd, user=run_user, quiet=False) + if data: data_dict = json.loads(data) return data_dict["beans"][0][property] except: + Logger.exception("Getting jmx metrics from NN failed. URL: " + str(qry)) return None \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/2209b720/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py b/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py index ab53ba7..0d9e28b 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py @@ -31,7 +31,7 @@ NAMENODE_HTTP_FRAGMENT = 'dfs.namenode.http-address.{0}.{1}' NAMENODE_HTTPS_FRAGMENT = 'dfs.namenode.https-address.{0}.{1}' JMX_URI_FRAGMENT = "http://{0}/jmx?qry=Hadoop:service=NameNode,name=NameNodeStatus" -def get_namenode_states(hdfs_site): +def get_namenode_states(hdfs_site, security_enabled, run_user): """ return format [('nn1', 'hdfs://hostname1:port1'), ('nn2', 'hdfs://hostname2:port2')] , [....], [....] """ @@ -59,7 +59,7 @@ def get_namenode_states(hdfs_site): value = str(hdfs_site[key]) jmx_uri = JMX_URI_FRAGMENT.format(value) - state = get_value_from_jmx(jmx_uri,'State') + state = get_value_from_jmx(jmx_uri, 'State', security_enabled, run_user, is_https_enabled) if state == HDFS_NN_STATE_ACTIVE: active_namenodes.append((nn_unique_id, value)) @@ -86,17 +86,17 @@ def is_ha_enabled(hdfs_site): return False -def get_active_namenode(hdfs_site): +def get_active_namenode(hdfs_site, security_enabled, run_user): """ return format is nn_unique_id and it's address ('nn1', 'hdfs://hostname1:port1') """ - active_namenodes = get_namenode_states(hdfs_site)[0] + active_namenodes = get_namenode_states(hdfs_site, security_enabled, run_user)[0] if active_namenodes: return active_namenodes[0] else: return UnknownConfiguration('fs_root') -def get_property_for_active_namenode(hdfs_site, property_name): +def get_property_for_active_namenode(hdfs_site, property_name, security_enabled, run_user): """ For dfs.namenode.rpc-address: - In non-ha mode it will return hdfs_site[dfs.namenode.rpc-address] @@ -104,7 +104,7 @@ def get_property_for_active_namenode(hdfs_site, property_name): """ if is_ha_enabled(hdfs_site): name_service = hdfs_site['dfs.nameservices'] - active_namenodes = get_namenode_states(hdfs_site)[0] + active_namenodes = get_namenode_states(hdfs_site, security_enabled, run_user)[0] if not len(active_namenodes): raise Fail("There is no active namenodes.") @@ -114,4 +114,4 @@ def get_property_for_active_namenode(hdfs_site, property_name): return hdfs_site[format("{property_name}.{name_service}.{active_namenode_id}")] else: return hdfs_site[property_name] - \ No newline at end of file + http://git-wip-us.apache.org/repos/asf/ambari/blob/2209b720/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py index 4b1c27c..f9c512f 100644 --- a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py +++ b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py @@ -120,8 +120,10 @@ class HdfsResourceJar: class WebHDFSUtil: def __init__(self, hdfs_site, run_user, security_enabled, logoutput=None): - https_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.https-address') - http_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.http-address') + https_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.https-address', + security_enabled, run_user) + http_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.http-address', + security_enabled, run_user) self.is_https_enabled = hdfs_site['dfs.https.enable'] if not is_empty(hdfs_site['dfs.https.enable']) else False address = https_nn_address if self.is_https_enabled else http_nn_address http://git-wip-us.apache.org/repos/asf/ambari/blob/2209b720/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode_ha_state.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode_ha_state.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode_ha_state.py index 50cfc0c..f3185bf 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode_ha_state.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode_ha_state.py @@ -81,7 +81,7 @@ class NamenodeHAState: raise Exception("Could not retrieve hostname from address " + actual_value) jmx_uri = jmx_uri_fragment.format(actual_value) - state = get_value_from_jmx(jmx_uri, "State") + state = get_value_from_jmx(jmx_uri, "State", params.security_enabled, params.hdfs_user, params.is_https_enabled) if not state: raise Exception("Could not retrieve Namenode state from URL " + jmx_uri) @@ -175,4 +175,4 @@ class NamenodeHAState: """ active_hosts = self.namenode_state_to_hostnames[NAMENODE_STATE.ACTIVE] if NAMENODE_STATE.ACTIVE in self.namenode_state_to_hostnames else [] standby_hosts = self.namenode_state_to_hostnames[NAMENODE_STATE.STANDBY] if NAMENODE_STATE.STANDBY in self.namenode_state_to_hostnames else [] - return len(active_hosts) == 1 and len(standby_hosts) == 1 \ No newline at end of file + return len(active_hosts) == 1 and len(standby_hosts) == 1 http://git-wip-us.apache.org/repos/asf/ambari/blob/2209b720/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py index 077a4f5..c6b10b8 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py @@ -37,6 +37,7 @@ from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.functions.format_jvm_option import format_jvm_option from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages +from resource_management.libraries.functions.is_empty import is_empty config = Script.get_config() @@ -384,6 +385,9 @@ policy_user = config['configurations']['ranger-hdfs-plugin-properties']['policy_ jdk_location = config['hostLevelParams']['jdk_location'] java_share_dir = '/usr/share/java' +is_https_enabled = config['configurations']['hdfs-site']['dfs.https.enabled'] if \ + not is_empty(config['configurations']['hdfs-site']['dfs.https.enabled']) else False + if has_ranger_admin: enable_ranger_hdfs = (config['configurations']['ranger-hdfs-plugin-properties']['ranger-hdfs-plugin-enabled'].lower() == 'yes') xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) http://git-wip-us.apache.org/repos/asf/ambari/blob/2209b720/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py index 39d2f10..646432d 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py @@ -283,7 +283,7 @@ class TestJournalnode(RMFTestCase): namenode_status_standby = open(namenode_status_standby_file, 'r').read() url_stream_mock = MagicMock() - url_stream_mock.read.side_effect = [namenode_status_active, namenode_status_standby] + (num_journalnodes * [namenode_jmx, journalnode_jmx]) + url_stream_mock.read.side_effect = (num_journalnodes * [namenode_jmx, journalnode_jmx]) urlopen_mock.return_value = url_stream_mock @@ -291,6 +291,7 @@ class TestJournalnode(RMFTestCase): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/journalnode.py", classname = "JournalNode", command = "post_rolling_restart", config_file = "journalnode-upgrade.json", + checked_call_mocks = [(0, str(namenode_status_active)), (0, str(namenode_status_standby))], hdp_stack_version = self.UPGRADE_STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES ) @@ -299,7 +300,7 @@ class TestJournalnode(RMFTestCase): urlopen_mock.assert_called_with("http://c6407.ambari.apache.org:8480/jmx") url_stream_mock.reset_mock() - url_stream_mock.read.side_effect = [namenode_status_active, namenode_status_standby] + (num_journalnodes * [namenode_jmx, journalnode_jmx]) + url_stream_mock.read.side_effect = (num_journalnodes * [namenode_jmx, journalnode_jmx]) urlopen_mock.return_value = url_stream_mock @@ -307,6 +308,7 @@ class TestJournalnode(RMFTestCase): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/journalnode.py", classname = "JournalNode", command = "post_rolling_restart", config_file = "journalnode-upgrade-hdfs-secure.json", + checked_call_mocks = [(0, str(namenode_status_active)), (0, str(namenode_status_standby))], hdp_stack_version = self.UPGRADE_STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES )