Repository: ambari Updated Branches: refs/heads/trunk 53b3ecfd9 -> 6ba9bd0a7
AMBARI-19629. Move schemaTool invocation out of configure() to hive_metastore start(). (Attila Doroszlai via stoader) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6ba9bd0a Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6ba9bd0a Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6ba9bd0a Branch: refs/heads/trunk Commit: 6ba9bd0a7d5534838a7c1679675cf1848806990f Parents: 53b3ecf Author: Attila Doroszlai <adorosz...@hortonworks.com> Authored: Fri Jan 20 06:55:14 2017 +0100 Committer: Toader, Sebastian <stoa...@hortonworks.com> Committed: Fri Jan 20 06:55:14 2017 +0100 ---------------------------------------------------------------------- .../HIVE/0.12.0.2.0/package/scripts/hive.py | 57 ++++++++++---------- .../package/scripts/hive_metastore.py | 5 +- .../stacks/2.0.6/HIVE/test_hive_metastore.py | 9 ++-- .../stacks/2.1/HIVE/test_hive_metastore.py | 14 ++--- 4 files changed, 46 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/6ba9bd0a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py index 16273c7..2a4fd30 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py @@ -121,7 +121,7 @@ def hive(name=None): if name == 'hiveserver2': setup_hiveserver2() if name == 'metastore': - setup_metastore() # schematool work + setup_metastore() def setup_hiveserver2(): import params @@ -299,32 +299,35 @@ def setup_metastore(): mode=0755, content=StaticFile('startMetastore.sh') ) - if params.init_metastore_schema: - create_schema_cmd = format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " - "{hive_schematool_bin}/schematool -initSchema " - "-dbType {hive_metastore_db_type} " - "-userName {hive_metastore_user_name} " - "-passWord {hive_metastore_user_passwd!p} -verbose") - - check_schema_created_cmd = as_user(format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " - "{hive_schematool_bin}/schematool -info " - "-dbType {hive_metastore_db_type} " - "-userName {hive_metastore_user_name} " - "-passWord {hive_metastore_user_passwd!p} -verbose"), params.hive_user) - - # HACK: in cases with quoted passwords and as_user (which does the quoting as well) !p won't work for hiding passwords. - # Fixing it with the hack below: - quoted_hive_metastore_user_passwd = quote_bash_args(quote_bash_args(params.hive_metastore_user_passwd)) - if quoted_hive_metastore_user_passwd[0] == "'" and quoted_hive_metastore_user_passwd[-1] == "'" \ - or quoted_hive_metastore_user_passwd[0] == '"' and quoted_hive_metastore_user_passwd[-1] == '"': - quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[1:-1] - Logger.sensitive_strings[repr(check_schema_created_cmd)] = repr(check_schema_created_cmd.replace( - format("-passWord {quoted_hive_metastore_user_passwd}"), "-passWord " + utils.PASSWORDS_HIDE_STRING)) - - Execute(create_schema_cmd, - not_if = check_schema_created_cmd, - user = params.hive_user - ) + +def create_metastore_schema(): + import params + + create_schema_cmd = format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " + "{hive_schematool_bin}/schematool -initSchema " + "-dbType {hive_metastore_db_type} " + "-userName {hive_metastore_user_name} " + "-passWord {hive_metastore_user_passwd!p} -verbose") + + check_schema_created_cmd = as_user(format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " + "{hive_schematool_bin}/schematool -info " + "-dbType {hive_metastore_db_type} " + "-userName {hive_metastore_user_name} " + "-passWord {hive_metastore_user_passwd!p} -verbose"), params.hive_user) + + # HACK: in cases with quoted passwords and as_user (which does the quoting as well) !p won't work for hiding passwords. + # Fixing it with the hack below: + quoted_hive_metastore_user_passwd = quote_bash_args(quote_bash_args(params.hive_metastore_user_passwd)) + if quoted_hive_metastore_user_passwd[0] == "'" and quoted_hive_metastore_user_passwd[-1] == "'" \ + or quoted_hive_metastore_user_passwd[0] == '"' and quoted_hive_metastore_user_passwd[-1] == '"': + quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[1:-1] + Logger.sensitive_strings[repr(check_schema_created_cmd)] = repr(check_schema_created_cmd.replace( + format("-passWord {quoted_hive_metastore_user_passwd}"), "-passWord " + utils.PASSWORDS_HIDE_STRING)) + + Execute(create_schema_cmd, + not_if = check_schema_created_cmd, + user = params.hive_user + ) """ Writes configuration files required by Hive. http://git-wip-us.apache.org/repos/asf/ambari/blob/6ba9bd0a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py index d1ecd65..8b69e45 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py @@ -36,8 +36,7 @@ from resource_management.libraries.functions.security_commons import validate_se from resource_management.libraries.functions.security_commons import FILE_TYPE_XML from resource_management.core.resources.system import File -from hive import hive -from hive import jdbc_connector +from hive import create_metastore_schema, hive, jdbc_connector from hive_service import hive_service from ambari_commons.os_family_impl import OsFamilyImpl from ambari_commons import OSConst @@ -57,6 +56,8 @@ class HiveMetastore(Script): # writing configurations on start required for securtity self.configure(env) + if params.init_metastore_schema: + create_metastore_schema() # execute without config lock hive_service('metastore', action='start', upgrade_type=upgrade_type) http://git-wip-us.apache.org/repos/asf/ambari/blob/6ba9bd0a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py index 0a98894..9c04acc 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py @@ -36,6 +36,7 @@ class TestHiveMetastore(RMFTestCase): target = RMFTestCase.TARGET_COMMON_SERVICES ) self.assert_configure_default() + self.assertNoMoreResources() def test_start_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py", @@ -47,6 +48,7 @@ class TestHiveMetastore(RMFTestCase): ) self.assert_configure_default() + self.assert_init_schema() self.assertResourceCalled('Execute', '/tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.err /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive', environment = {'HADOOP_HOME': '/usr/hdp/current/hadoop-client', 'HIVE_BIN': 'hive', @@ -107,6 +109,7 @@ class TestHiveMetastore(RMFTestCase): target = RMFTestCase.TARGET_COMMON_SERVICES ) self.assert_configure_secured() + self.assert_init_schema() self.assertResourceCalled('Execute', '/tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.err /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive', environment = {'HADOOP_HOME': '/usr/hdp/current/hadoop-client', 'HIVE_BIN': 'hive', @@ -264,6 +267,8 @@ class TestHiveMetastore(RMFTestCase): content = StaticFile('startMetastore.sh'), mode = 0755, ) + + def assert_init_schema(self): self.assertResourceCalled('Execute', 'export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -initSchema -dbType mysql -userName hive -passWord \'!`"\'"\'"\' 1\' -verbose', not_if = 'ambari-sudo.sh su hive -l -s /bin/bash -c \'[RMF_EXPORT_PLACEHOLDER]export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -info -dbType mysql -userName hive -passWord \'"\'"\'!`"\'"\'"\'"\'"\'"\'"\'"\'"\' 1\'"\'"\' -verbose\'', user = 'hive', @@ -387,10 +392,6 @@ class TestHiveMetastore(RMFTestCase): content = StaticFile('startMetastore.sh'), mode = 0755, ) - self.assertResourceCalled('Execute', 'export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -initSchema -dbType mysql -userName hive -passWord \'!`"\'"\'"\' 1\' -verbose', - not_if = 'ambari-sudo.sh su hive -l -s /bin/bash -c \'[RMF_EXPORT_PLACEHOLDER]export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -info -dbType mysql -userName hive -passWord \'"\'"\'!`"\'"\'"\'"\'"\'"\'"\'"\'"\' 1\'"\'"\' -verbose\'', - user = 'hive', - ) @patch("resource_management.core.shell.call") @patch("resource_management.libraries.functions.get_stack_version") http://git-wip-us.apache.org/repos/asf/ambari/blob/6ba9bd0a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py index f9e910b..a159cd0 100644 --- a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py +++ b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py @@ -42,6 +42,7 @@ class TestHiveMetastore(RMFTestCase): target = RMFTestCase.TARGET_COMMON_SERVICES ) self.assert_configure_default() + self.assertNoMoreResources() def test_start_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py", @@ -53,6 +54,7 @@ class TestHiveMetastore(RMFTestCase): ) self.assert_configure_default() + self.assert_init_schema('aaa') self.assertResourceCalled('Execute', '/tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.err /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive', environment = {'HADOOP_HOME': '/usr/hdp/current/hadoop-client', @@ -83,6 +85,7 @@ class TestHiveMetastore(RMFTestCase): ) self.assert_configure_default() + self.assert_init_schema('aaa') self.assertResourceCalled('Execute', '/tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.err /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive', environment = {'HADOOP_HOME': '/usr/hdp/current/hadoop-client', @@ -146,6 +149,7 @@ class TestHiveMetastore(RMFTestCase): ) self.assert_configure_secured() + self.assert_init_schema('asd') self.assertResourceCalled('Execute', '/tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.err /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive', environment = {'HADOOP_HOME': '/usr/hdp/current/hadoop-client', 'HIVE_BIN': 'hive', @@ -287,8 +291,10 @@ class TestHiveMetastore(RMFTestCase): content = StaticFile('startMetastore.sh'), mode = 0755, ) - self.assertResourceCalled('Execute', 'export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -initSchema -dbType mysql -userName hive -passWord aaa -verbose', - not_if = "ambari-sudo.sh su hive -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -info -dbType mysql -userName hive -passWord aaa -verbose'", + + def assert_init_schema(self, password): + self.assertResourceCalled('Execute', 'export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -initSchema -dbType mysql -userName hive -passWord {password} -verbose'.format(password = password), + not_if = "ambari-sudo.sh su hive -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -info -dbType mysql -userName hive -passWord {password} -verbose'".format(password = password), user = 'hive', ) @@ -399,10 +405,6 @@ class TestHiveMetastore(RMFTestCase): content = StaticFile('startMetastore.sh'), mode = 0755, ) - self.assertResourceCalled('Execute', 'export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -initSchema -dbType mysql -userName hive -passWord asd -verbose', - not_if = "ambari-sudo.sh su hive -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -info -dbType mysql -userName hive -passWord asd -verbose'", - user = 'hive', - ) @patch("resource_management.libraries.functions.security_commons.build_expectations") @patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem")
