Repository: ambari Updated Branches: refs/heads/branch-2.5 a03389c75 -> 9a555e7b1
AMBARI-19629. Move schemaTool invocation out of configure() to hive_metastore start(). (Attila Doroszlai via stoader) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9a555e7b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9a555e7b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9a555e7b Branch: refs/heads/branch-2.5 Commit: 9a555e7b14d8f6a85a052e94452f9376977b18f7 Parents: a03389c Author: Attila Doroszlai <adorosz...@hortonworks.com> Authored: Fri Jan 20 07:05:58 2017 +0100 Committer: Toader, Sebastian <stoa...@hortonworks.com> Committed: Fri Jan 20 07:05:58 2017 +0100 ---------------------------------------------------------------------- .../HIVE/0.12.0.2.0/package/scripts/hive.py | 55 +++++++++++--------- .../package/scripts/hive_metastore.py | 5 +- .../stacks/2.0.6/HIVE/test_hive_metastore.py | 25 ++++----- .../stacks/2.1/HIVE/test_hive_metastore.py | 18 ++++--- 4 files changed, 55 insertions(+), 48 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9a555e7b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py index 99980b3..a2879da 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py @@ -309,32 +309,6 @@ def hive(name=None): mode=0755, content=StaticFile('startMetastore.sh') ) - if params.init_metastore_schema: - create_schema_cmd = format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " - "{hive_schematool_bin}/schematool -initSchema " - "-dbType {hive_metastore_db_type} " - "-userName {hive_metastore_user_name} " - "-passWord {hive_metastore_user_passwd!p} -verbose") - - check_schema_created_cmd = as_user(format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " - "{hive_schematool_bin}/schematool -info " - "-dbType {hive_metastore_db_type} " - "-userName {hive_metastore_user_name} " - "-passWord {hive_metastore_user_passwd!p} -verbose"), params.hive_user) - - # HACK: in cases with quoted passwords and as_user (which does the quoting as well) !p won't work for hiding passwords. - # Fixing it with the hack below: - quoted_hive_metastore_user_passwd = quote_bash_args(quote_bash_args(params.hive_metastore_user_passwd)) - if quoted_hive_metastore_user_passwd[0] == "'" and quoted_hive_metastore_user_passwd[-1] == "'" \ - or quoted_hive_metastore_user_passwd[0] == '"' and quoted_hive_metastore_user_passwd[-1] == '"': - quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[1:-1] - Logger.sensitive_strings[repr(check_schema_created_cmd)] = repr(check_schema_created_cmd.replace( - format("-passWord {quoted_hive_metastore_user_passwd}"), "-passWord " + utils.PASSWORDS_HIDE_STRING)) - - Execute(create_schema_cmd, - not_if = check_schema_created_cmd, - user = params.hive_user - ) elif name == 'hiveserver2': File(params.start_hiveserver2_path, mode=0755, @@ -368,6 +342,35 @@ def hive(name=None): group=params.user_group, mode=0755) +def create_metastore_schema(): + import params + + create_schema_cmd = format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " + "{hive_schematool_bin}/schematool -initSchema " + "-dbType {hive_metastore_db_type} " + "-userName {hive_metastore_user_name} " + "-passWord {hive_metastore_user_passwd!p} -verbose") + + check_schema_created_cmd = as_user(format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " + "{hive_schematool_bin}/schematool -info " + "-dbType {hive_metastore_db_type} " + "-userName {hive_metastore_user_name} " + "-passWord {hive_metastore_user_passwd!p} -verbose"), params.hive_user) + + # HACK: in cases with quoted passwords and as_user (which does the quoting as well) !p won't work for hiding passwords. + # Fixing it with the hack below: + quoted_hive_metastore_user_passwd = quote_bash_args(quote_bash_args(params.hive_metastore_user_passwd)) + if quoted_hive_metastore_user_passwd[0] == "'" and quoted_hive_metastore_user_passwd[-1] == "'" \ + or quoted_hive_metastore_user_passwd[0] == '"' and quoted_hive_metastore_user_passwd[-1] == '"': + quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[1:-1] + Logger.sensitive_strings[repr(check_schema_created_cmd)] = repr(check_schema_created_cmd.replace( + format("-passWord {quoted_hive_metastore_user_passwd}"), "-passWord " + utils.PASSWORDS_HIDE_STRING)) + + Execute(create_schema_cmd, + not_if = check_schema_created_cmd, + user = params.hive_user + ) + """ Writes configuration files required by Hive. """ http://git-wip-us.apache.org/repos/asf/ambari/blob/9a555e7b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py index 17bf581..79d886d 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py @@ -36,8 +36,7 @@ from resource_management.libraries.functions.security_commons import validate_se from resource_management.libraries.functions.security_commons import FILE_TYPE_XML from resource_management.core.resources.system import File -from hive import hive -from hive import jdbc_connector +from hive import create_metastore_schema, hive, jdbc_connector from hive_service import hive_service from ambari_commons.os_family_impl import OsFamilyImpl from ambari_commons import OSConst @@ -57,6 +56,8 @@ class HiveMetastore(Script): # writing configurations on start required for securtity self.configure(env) + if params.init_metastore_schema: + create_metastore_schema() hive_service('metastore', action='start', upgrade_type=upgrade_type) http://git-wip-us.apache.org/repos/asf/ambari/blob/9a555e7b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py index 821248e..e942201 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py @@ -36,6 +36,7 @@ class TestHiveMetastore(RMFTestCase): target = RMFTestCase.TARGET_COMMON_SERVICES ) self.assert_configure_default() + self.assertNoMoreResources() def test_start_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py", @@ -47,6 +48,7 @@ class TestHiveMetastore(RMFTestCase): ) self.assert_configure_default() + self.assert_init_schema() self.assertResourceCalled('Execute', '/tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.err /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive', environment = {'HADOOP_HOME': '/usr/hdp/current/hadoop-client', 'HIVE_BIN': 'hive', @@ -107,6 +109,7 @@ class TestHiveMetastore(RMFTestCase): target = RMFTestCase.TARGET_COMMON_SERVICES ) self.assert_configure_secured() + self.assert_init_schema() self.assertResourceCalled('Execute', '/tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.err /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive', environment = {'HADOOP_HOME': '/usr/hdp/current/hadoop-client', 'HIVE_BIN': 'hive', @@ -240,10 +243,6 @@ class TestHiveMetastore(RMFTestCase): content = StaticFile('startMetastore.sh'), mode = 0755, ) - self.assertResourceCalled('Execute', 'export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -initSchema -dbType mysql -userName hive -passWord \'!`"\'"\'"\' 1\' -verbose', - not_if = 'ambari-sudo.sh su hive -l -s /bin/bash -c \'[RMF_EXPORT_PLACEHOLDER]export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -info -dbType mysql -userName hive -passWord \'"\'"\'!`"\'"\'"\'"\'"\'"\'"\'"\'"\' 1\'"\'"\' -verbose\'', - user = 'hive', - ) self.assertResourceCalled('Directory', '/var/run/hive', owner = 'hive', group = 'hadoop', @@ -363,10 +362,6 @@ class TestHiveMetastore(RMFTestCase): content = StaticFile('startMetastore.sh'), mode = 0755, ) - self.assertResourceCalled('Execute', 'export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -initSchema -dbType mysql -userName hive -passWord \'!`"\'"\'"\' 1\' -verbose', - not_if = 'ambari-sudo.sh su hive -l -s /bin/bash -c \'[RMF_EXPORT_PLACEHOLDER]export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -info -dbType mysql -userName hive -passWord \'"\'"\'!`"\'"\'"\'"\'"\'"\'"\'"\'"\' 1\'"\'"\' -verbose\'', - user = 'hive', - ) self.assertResourceCalled('Directory', '/var/run/hive', owner = 'hive', group = 'hadoop', @@ -389,6 +384,12 @@ class TestHiveMetastore(RMFTestCase): cd_access = 'a', ) + def assert_init_schema(self): + self.assertResourceCalled('Execute', 'export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -initSchema -dbType mysql -userName hive -passWord \'!`"\'"\'"\' 1\' -verbose', + not_if = 'ambari-sudo.sh su hive -l -s /bin/bash -c \'[RMF_EXPORT_PLACEHOLDER]export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -info -dbType mysql -userName hive -passWord \'"\'"\'!`"\'"\'"\'"\'"\'"\'"\'"\'"\' 1\'"\'"\' -verbose\'', + user = 'hive', + ) + @patch("resource_management.core.shell.call") @patch("resource_management.libraries.functions.get_stack_version") def test_start_ru(self, call_mock, get_stack_version_mock): @@ -510,10 +511,6 @@ class TestHiveMetastore(RMFTestCase): self.maxDiff = None - self.assertResourceCalled('Execute', 'export HIVE_CONF_DIR=/usr/hdp/current/hive-server2/conf/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -initSchema -dbType mysql -userName hive -passWord aaa -verbose', - not_if = "ambari-sudo.sh su hive -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]export HIVE_CONF_DIR=/usr/hdp/current/hive-server2/conf/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -info -dbType mysql -userName hive -passWord aaa -verbose'", - user = 'hive') - self.assertResourceCalled('Directory', '/var/run/hive', owner = 'hive', group = 'hadoop', @@ -535,6 +532,10 @@ class TestHiveMetastore(RMFTestCase): create_parents = True, cd_access = 'a') + self.assertResourceCalled('Execute', 'export HIVE_CONF_DIR=/usr/hdp/current/hive-server2/conf/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -initSchema -dbType mysql -userName hive -passWord aaa -verbose', + not_if = "ambari-sudo.sh su hive -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]export HIVE_CONF_DIR=/usr/hdp/current/hive-server2/conf/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -info -dbType mysql -userName hive -passWord aaa -verbose'", + user = 'hive') + self.assertResourceCalled('Execute', '/tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.err /var/run/hive/hive.pid /usr/hdp/current/hive-server2/conf/conf.server /var/log/hive', environment = {'HADOOP_HOME': '/usr/hdp/2.3.0.0-1234/hadoop', 'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45', 'HIVE_BIN': '/usr/hdp/current/hive-server2/bin/hive'}, not_if = None, http://git-wip-us.apache.org/repos/asf/ambari/blob/9a555e7b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py index a904bc2..de2dce8 100644 --- a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py +++ b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py @@ -42,6 +42,7 @@ class TestHiveMetastore(RMFTestCase): target = RMFTestCase.TARGET_COMMON_SERVICES ) self.assert_configure_default() + self.assertNoMoreResources() def test_start_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py", @@ -53,6 +54,7 @@ class TestHiveMetastore(RMFTestCase): ) self.assert_configure_default() + self.assert_init_schema('aaa') self.assertResourceCalled('Execute', '/tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.err /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive', environment = {'HADOOP_HOME': '/usr/hdp/current/hadoop-client', @@ -83,6 +85,7 @@ class TestHiveMetastore(RMFTestCase): ) self.assert_configure_default() + self.assert_init_schema('aaa') self.assertResourceCalled('Execute', '/tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.err /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive', environment = {'HADOOP_HOME': '/usr/hdp/current/hadoop-client', @@ -146,6 +149,7 @@ class TestHiveMetastore(RMFTestCase): ) self.assert_configure_secured() + self.assert_init_schema('asd') self.assertResourceCalled('Execute', '/tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.err /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive', environment = {'HADOOP_HOME': '/usr/hdp/current/hadoop-client', 'HIVE_BIN': 'hive', @@ -266,10 +270,6 @@ class TestHiveMetastore(RMFTestCase): content = StaticFile('startMetastore.sh'), mode = 0755, ) - self.assertResourceCalled('Execute', 'export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -initSchema -dbType mysql -userName hive -passWord aaa -verbose', - not_if = "ambari-sudo.sh su hive -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -info -dbType mysql -userName hive -passWord aaa -verbose'", - user = 'hive', - ) self.assertResourceCalled('Directory', '/var/run/hive', owner = 'hive', group = 'hadoop', @@ -379,10 +379,6 @@ class TestHiveMetastore(RMFTestCase): content = StaticFile('startMetastore.sh'), mode = 0755, ) - self.assertResourceCalled('Execute', 'export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -initSchema -dbType mysql -userName hive -passWord asd -verbose', - not_if = "ambari-sudo.sh su hive -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -info -dbType mysql -userName hive -passWord asd -verbose'", - user = 'hive', - ) self.assertResourceCalled('Directory', '/var/run/hive', owner = 'hive', group = 'hadoop', @@ -405,6 +401,12 @@ class TestHiveMetastore(RMFTestCase): cd_access = 'a', ) + def assert_init_schema(self, password): + self.assertResourceCalled('Execute', 'export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -initSchema -dbType mysql -userName hive -passWord {password} -verbose'.format(password = password), + not_if = "ambari-sudo.sh su hive -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -info -dbType mysql -userName hive -passWord {password} -verbose'".format(password = password), + user = 'hive', + ) + @patch("resource_management.libraries.functions.security_commons.build_expectations") @patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem") @patch("resource_management.libraries.functions.security_commons.validate_security_config_properties")