AMBARI-20200: hive-site.xml, hbase-site.xml, etc. are not found in class path for Zeppelin (Prabhjyot Singh via r-kamath)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6025d413 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6025d413 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6025d413 Branch: refs/heads/branch-feature-AMBARI-12556 Commit: 6025d413da204a4cf5cf69fe90a413b905ac9344 Parents: 109bba1 Author: Renjith Kamath <renjith.kam...@gmail.com> Authored: Mon Mar 6 15:56:19 2017 +0530 Committer: Renjith Kamath <renjith.kam...@gmail.com> Committed: Mon Mar 6 15:56:19 2017 +0530 ---------------------------------------------------------------------- .../0.6.0.2.5/configuration/zeppelin-env.xml | 2 +- .../0.6.0.2.5/package/scripts/master.py | 36 +++++++++++++++----- .../0.6.0.2.5/package/scripts/params.py | 8 +++++ .../stacks/2.5/ZEPPELIN/test_zeppelin_master.py | 26 +++++++------- 4 files changed, 49 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/6025d413/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml index 677158c..bdef319 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml @@ -89,7 +89,7 @@ export ZEPPELIN_PID_DIR={{zeppelin_pid_dir}} # export ZEPPELIN_NOTEBOOK_STORAGE # Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote). # export ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC # If there are multiple notebook storages, should we treat the first one as the only source of truth? # export ZEPPELIN_NOTEBOOK_PUBLIC # Make notebook public by default when created, private otherwise - +export ZEPPELIN_INTP_CLASSPATH_OVERRIDES="{{external_dependency_conf}}" #### Spark interpreter configuration #### ## Use provided spark installation ## http://git-wip-us.apache.org/repos/asf/ambari/blob/6025d413/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py index 8a1fad6..fc24918 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py @@ -106,6 +106,17 @@ class Master(Script): mode=0755 ) + def create_zeppelin_hdfs_conf_dir(self, env): + import params + env.set_params(params) + Directory([params.external_dependency_conf], + owner=params.zeppelin_user, + group=params.zeppelin_group, + cd_access="a", + create_parents=True, + mode=0755 + ) + def chown_zeppelin_pid_dir(self, env): import params env.set_params(params) @@ -150,17 +161,26 @@ class Master(Script): File(format("{params.conf_dir}/log4j.properties"), content=params.log4j_properties_content, owner=params.zeppelin_user, group=params.zeppelin_group) + self.create_zeppelin_hdfs_conf_dir(env) # copy hive-site.xml only if Spark 1.x is installed - if 'spark-defaults' in params.config['configurations'] and \ - os.path.exists("/etc/spark/conf/hive-site.xml"): - File(format("{params.conf_dir}/hive-site.xml"), content=StaticFile("/etc/spark/conf/hive-site.xml"), - owner=params.zeppelin_user, group=params.zeppelin_group) + if 'spark-defaults' in params.config['configurations'] and params.is_hive_installed: + XmlConfig("hive-site.xml", + conf_dir=params.external_dependency_conf, + configurations=params.spark_hive_properties, + owner=params.zeppelin_user, + group=params.zeppelin_group, + mode=0644) - if len(params.hbase_master_hosts) > 0 and \ - os.path.exists("/etc/hbase/conf/hbase-site.xml"): + if len(params.hbase_master_hosts) > 0 and params.is_hbase_installed: # copy hbase-site.xml - File(format("{params.conf_dir}/hbase-site.xml"), content=StaticFile("/etc/hbase/conf/hbase-site.xml"), - owner=params.zeppelin_user, group=params.zeppelin_group) + XmlConfig("hbase-site.xml", + conf_dir=params.external_dependency_conf, + configurations=params.config['configurations']['hbase-site'], + configuration_attributes=params.config['configuration_attributes']['hbase-site'], + owner=params.zeppelin_user, + group=params.zeppelin_group, + mode=0644 + ) def stop(self, env, upgrade_type=None): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/6025d413/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py index 16a2782..9fd105b 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py @@ -95,6 +95,7 @@ zeppelin_hdfs_user_dir = format("/user/{zeppelin_user}") zeppelin_dir = os.path.join(*[install_dir, zeppelin_dirname]) conf_dir = "/etc/zeppelin/conf" +external_dependency_conf = "/etc/zeppelin/conf/external-dependency-conf" notebook_dir = os.path.join(*[install_dir, zeppelin_dirname, 'notebook']) # zeppelin-env.sh @@ -121,7 +122,12 @@ hive_metastore_port = None hive_server_port = None hive_zookeeper_quorum = None hive_server2_support_dynamic_service_discovery = None +is_hive_installed = False if 'hive_server_host' in master_configs and len(master_configs['hive_server_host']) != 0: + is_hive_installed = True + spark_hive_properties = { + 'hive.metastore.uris': config['configurations']['hive-site']['hive.metastore.uris'] + } hive_server_host = str(master_configs['hive_server_host'][0]) hive_metastore_host = str(master_configs['hive_metastore_host'][0]) hive_metastore_port = str( @@ -133,7 +139,9 @@ if 'hive_server_host' in master_configs and len(master_configs['hive_server_host # detect hbase details if installed zookeeper_znode_parent = None hbase_zookeeper_quorum = None +is_hbase_installed = False if 'hbase_master_hosts' in master_configs and 'hbase-site' in config['configurations']: + is_hbase_installed = True zookeeper_znode_parent = config['configurations']['hbase-site']['zookeeper.znode.parent'] hbase_zookeeper_quorum = config['configurations']['hbase-site']['hbase.zookeeper.quorum'] http://git-wip-us.apache.org/repos/asf/ambari/blob/6025d413/ambari-server/src/test/python/stacks/2.5/ZEPPELIN/test_zeppelin_master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/ZEPPELIN/test_zeppelin_master.py b/ambari-server/src/test/python/stacks/2.5/ZEPPELIN/test_zeppelin_master.py index bd0f5ef..60c18c9 100644 --- a/ambari-server/src/test/python/stacks/2.5/ZEPPELIN/test_zeppelin_master.py +++ b/ambari-server/src/test/python/stacks/2.5/ZEPPELIN/test_zeppelin_master.py @@ -75,10 +75,12 @@ class TestZeppelinMaster(RMFTestCase): content = u'log4j.rootLogger = INFO, dailyfile', group = u'zeppelin', ) - self.assertResourceCalled('File', '/etc/zeppelin/conf/hive-site.xml', + self.assertResourceCalled('Directory', '/etc/zeppelin/conf/external-dependency-conf', owner = 'zeppelin', - content = StaticFile('/etc/spark/conf/hive-site.xml'), group = 'zeppelin', + create_parents = True, + mode = 0755, + cd_access = 'a', ) def assert_configure_secured(self): @@ -125,15 +127,15 @@ class TestZeppelinMaster(RMFTestCase): content = u'log4j.rootLogger = INFO, dailyfile', group = u'zeppelin', ) - self.assertResourceCalled('File', '/etc/zeppelin/conf/hive-site.xml', + self.assertResourceCalled('Directory', '/etc/zeppelin/conf/external-dependency-conf', owner = 'zeppelin', - content = StaticFile('/etc/spark/conf/hive-site.xml'), group = 'zeppelin', + create_parents = True, + mode = 0755, + cd_access = 'a', ) - @patch('os.path.exists') - def test_configure_default(self, os_path_exists_mock): - os_path_exists_mock.side_effect = lambda path: path == '/etc/spark/conf/hive-site.xml' + def test_configure_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/master.py", classname = "Master", command = "configure", @@ -144,9 +146,7 @@ class TestZeppelinMaster(RMFTestCase): self.assert_configure_default() self.assertNoMoreResources() - @patch('os.path.exists') - def test_configure_secured(self, os_path_exists_mock): - os_path_exists_mock.side_effect = lambda path: path == '/etc/spark/conf/hive-site.xml' + def test_configure_secured(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/master.py", classname = "Master", command = "configure", @@ -202,10 +202,8 @@ class TestZeppelinMaster(RMFTestCase): user = 'zeppelin', ) self.assertNoMoreResources() - - @patch('os.path.exists') - def test_start_default(self, os_path_exists_mock): - os_path_exists_mock.side_effect = lambda path: path == '/etc/spark/conf/hive-site.xml' + + def test_start_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/master.py", classname = "Master", command = "start",
