Repository: ambari Updated Branches: refs/heads/branch-2.1.2 8a800741e -> 79dc49ae0
AMBARI-13459 - Hive Metastore Upgrade Retry Fails Because Of Driver Copy Issue (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/79dc49ae Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/79dc49ae Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/79dc49ae Branch: refs/heads/branch-2.1.2 Commit: 79dc49ae03d7c59568e445261132ae52ace8f719 Parents: 8a80074 Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Fri Oct 16 14:09:26 2015 -0400 Committer: Jonathan Hurley <jhur...@hortonworks.com> Committed: Fri Oct 16 16:08:51 2015 -0400 ---------------------------------------------------------------------- .../package/scripts/hive_metastore.py | 30 +++++++++++++------- .../stacks/2.1/HIVE/test_hive_metastore.py | 8 ++++-- 2 files changed, 24 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/79dc49ae/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py index 796ec18..8a3833b 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py @@ -33,7 +33,6 @@ from resource_management.libraries.functions.security_commons import get_params_ from resource_management.libraries.functions.security_commons import validate_security_config_properties from resource_management.libraries.functions.security_commons import FILE_TYPE_XML from resource_management.core.resources.system import File -from resource_management.core.shell import as_sudo from hive import hive from hive import jdbc_connector @@ -155,7 +154,7 @@ class HiveMetastoreDefault(HiveMetastore): """ Executes the schema upgrade binary. This is its own function because it could be called as a standalone task from the upgrade pack, but is safe to run it for each - metastore instance. + metastore instance. The schema upgrade on an already upgraded metastore is a NOOP. The metastore schema upgrade requires a database driver library for most databases. During an upgrade, it's possible that the library is not present, @@ -173,28 +172,36 @@ class HiveMetastoreDefault(HiveMetastore): # present, then download it first if params.hive_jdbc_driver in params.hive_jdbc_drivers_list and params.hive_use_existing_db: target_directory = format("/usr/hdp/{version}/hive/lib") - if not os.path.exists(params.target): - # download it + + # normally, the JDBC driver would be referenced by /usr/hdp/current/.../foo.jar + # but if hdp-select is called and the restart fails, then this means that current pointer + # is now pointing to the upgraded version location; that's bad for the cp command + source_jdbc_file = format(params.target.replace("/usr/hdp/current", "/usr/hdp/{current_version}")) + + # download it if it does not exist + if not os.path.exists(source_jdbc_file): jdbc_connector() + target_directory_and_filename = os.path.join(target_directory, os.path.basename(source_jdbc_file)) + if params.sqla_db_used: target_native_libs_directory = format("{target_directory}/native/lib64") Execute(format("yes | {sudo} cp {jars_in_hive_lib} {target_directory}")) - Directory(target_native_libs_directory, - recursive=True) + Directory(target_native_libs_directory, recursive=True) Execute(format("yes | {sudo} cp {libs_in_hive_lib} {target_native_libs_directory}")) Execute(format("{sudo} chown -R {hive_user}:{user_group} {hive_lib}/*")) else: - Execute(('cp', params.target, target_directory), - path=["/bin", "/usr/bin/"], sudo = True) + # copy the JDBC driver from the older metastore location to the new location only + # if it does not already exist + if not os.path.exists(target_directory_and_filename): + Execute(('cp', source_jdbc_file, target_directory), + path=["/bin", "/usr/bin/"], sudo = True) - File(os.path.join(target_directory, os.path.basename(params.target)), - mode = 0644, - ) + File(target_directory_and_filename, mode = 0644) # build the schema tool command binary = format("/usr/hdp/{version}/hive/bin/schematool") @@ -215,5 +222,6 @@ class HiveMetastoreDefault(HiveMetastore): command = format("{binary} -dbType {hive_metastore_db_type} -upgradeSchema") Execute(command, user=params.hive_user, tries=1, environment=env_dict, logoutput=True) + if __name__ == "__main__": HiveMetastore().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/79dc49ae/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py index 9d42b9f..8150cdb 100644 --- a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py +++ b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py @@ -539,7 +539,7 @@ class TestHiveMetastore(RMFTestCase): get_hdp_version_mock.return_value = '2.3.0.0-1234' def side_effect(path): - if path == "/usr/hdp/current/hive-server2/lib/mysql-connector-java.jar": + if path == "/usr/hdp/2.2.7.0-1234/hive-server2/lib/mysql-connector-java.jar": return True return False @@ -554,6 +554,8 @@ class TestHiveMetastore(RMFTestCase): version = '2.3.0.0-1234' json_content['commandParams']['version'] = version json_content['hostLevelParams']['stack_version'] = "2.3" + json_content['hostLevelParams']['current_version'] = "2.2.7.0-1234" + # trigger the code to think it needs to copy the JAR json_content['configurations']['hive-site']['javax.jdo.option.ConnectionDriverName'] = "com.mysql.jdbc.Driver" @@ -570,7 +572,7 @@ class TestHiveMetastore(RMFTestCase): mocks_dict = mocks_dict) self.assertResourceCalled('Execute', - ('cp', '/usr/hdp/current/hive-server2/lib/mysql-connector-java.jar', '/usr/hdp/2.3.0.0-1234/hive/lib'), + ('cp', '/usr/hdp/2.2.7.0-1234/hive-server2/lib/mysql-connector-java.jar', '/usr/hdp/2.3.0.0-1234/hive/lib'), path = ['/bin', '/usr/bin/'], sudo = True) self.assertResourceCalled('File', '/usr/hdp/2.3.0.0-1234/hive/lib/mysql-connector-java.jar', @@ -578,7 +580,7 @@ class TestHiveMetastore(RMFTestCase): ) self.assertResourceCalled('Execute', "/usr/hdp/2.3.0.0-1234/hive/bin/schematool -dbType mysql -upgradeSchema", - logoutput = True, environment = {'HIVE_CONF_DIR': '/usr/hdp/current/hive-server2/conf/conf.server'}, + logoutput = True, environment = {'HIVE_CONF_DIR': '/etc/hive/conf.server'}, tries = 1, user = 'hive') self.assertResourceCalled('Execute', ('hdp-select', 'set', 'hive-metastore', version), sudo=True,)
