Repository: ambari Updated Branches: refs/heads/trunk cc6f4a682 -> 36a1c6699
AMBARI-11675 - Hive Upgrade Fails Because Of Missing Database Library (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/36a1c669 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/36a1c669 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/36a1c669 Branch: refs/heads/trunk Commit: 36a1c669970bf219c01157b40385218608685a38 Parents: cc6f4a6 Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Wed Jun 3 21:50:00 2015 -0400 Committer: Jonathan Hurley <jhur...@hortonworks.com> Committed: Thu Jun 4 11:20:33 2015 -0400 ---------------------------------------------------------------------- .../package/scripts/hive_metastore.py | 20 ++++++++ .../stacks/2.1/HIVE/test_hive_metastore.py | 52 ++++++++++++++++++++ 2 files changed, 72 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/36a1c669/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py index 1813ee3..e6f3e97 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py @@ -17,6 +17,8 @@ See the License for the specific language governing permissions and limitations under the License. """ +import os + from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute from resource_management.libraries.script import Script @@ -32,6 +34,7 @@ from resource_management.libraries.functions.security_commons import validate_se from resource_management.libraries.functions.security_commons import FILE_TYPE_XML from hive import hive +from hive import jdbc_connector from hive_service import hive_service from ambari_commons.os_family_impl import OsFamilyImpl from ambari_commons import OSConst @@ -149,11 +152,16 @@ class HiveMetastoreDefault(HiveMetastore): else: self.put_structured_out({"securityState": "UNSECURED"}) + def upgrade_schema(self, env): """ Executes the schema upgrade binary. This is its own function because it could be called as a standalone task from the upgrade pack, but is safe to run it for each metastore instance. + + The metastore schema upgrade requires a database driver library for most + databases. During an upgrade, it's possible that the library is not present, + so this will also attempt to copy/download the appropriate driver. """ Logger.info("Upgrading Hive Metastore") import params @@ -163,6 +171,18 @@ class HiveMetastoreDefault(HiveMetastore): kinit_command=format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal}; ") Execute(kinit_command,user=params.smokeuser) + # ensure that the JDBC drive is present for the schema tool; if it's not + # present, then download it first + if params.hive_jdbc_driver in params.hive_jdbc_drivers_list and params.hive_use_existing_db: + target_directory = format("/usr/hdp/{version}/hive/lib") + if not os.path.exists(params.target): + # download it + jdbc_connector() + + Execute(('cp', params.target, target_directory), + path=["/bin", "/usr/bin/"], sudo = True) + + # build the schema tool command binary = format("/usr/hdp/{version}/hive/bin/schematool") # the conf.server directory changed locations between HDP 2.2 and 2.3 http://git-wip-us.apache.org/repos/asf/ambari/blob/36a1c669/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py index 246f206..5b924ae 100644 --- a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py +++ b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py @@ -22,6 +22,7 @@ import os from mock.mock import MagicMock, call, patch from stacks.utils.RMFTestCase import * +@patch("platform.linux_distribution", new = MagicMock(return_value="Linux")) class TestHiveMetastore(RMFTestCase): COMMON_SERVICES_PACKAGE_DIR = "HIVE/0.12.0.2.0/package" STACK_VERSION = "2.0.6" @@ -525,3 +526,54 @@ class TestHiveMetastore(RMFTestCase): self.assertEquals( "conf-select set-conf-dir --package hive --stack-version 2.3.0.0-1234 --conf-version 0", mocks_dict['call'].call_args_list[1][0][0]) + + + @patch("os.path.exists") + @patch("resource_management.core.shell.call") + @patch("resource_management.libraries.functions.get_hdp_version") + def test_upgrade_metastore_schema(self, get_hdp_version_mock, call_mock, os_path_exists_mock): + + get_hdp_version_mock.return_value = '2.3.0.0-1234' + + def side_effect(path): + if path == "/usr/hdp/current/hive-server2/lib/mysql-connector-java.jar": + return True + return False + + os_path_exists_mock.side_effect = side_effect + + config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json" + + with open(config_file, "r") as f: + json_content = json.load(f) + + # must be HDP 2.3+ + version = '2.3.0.0-1234' + json_content['commandParams']['version'] = version + json_content['hostLevelParams']['stack_version'] = "2.3" + + # trigger the code to think it needs to copy the JAR + json_content['configurations']['hive-site']['javax.jdo.option.ConnectionDriverName'] = "com.mysql.jdbc.Driver" + json_content['configurations']['hive-env']['hive_database'] = "Existing" + + mocks_dict = {} + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py", + classname = "HiveMetastore", + command = "pre_rolling_restart", + config_dict = json_content, + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES, + call_mocks = [(0, None), (0, None)], + mocks_dict = mocks_dict) + + self.assertResourceCalled('Execute', + ('cp', '/usr/hdp/current/hive-server2/lib/mysql-connector-java.jar', '/usr/hdp/2.3.0.0-1234/hive/lib'), + path = ['/bin', '/usr/bin/'], sudo = True) + + self.assertResourceCalled('Execute', "/usr/hdp/2.3.0.0-1234/hive/bin/schematool -dbType mysql -upgradeSchema", + logoutput = True, environment = {'HIVE_CONF_DIR': '/usr/hdp/current/hive-server2/conf/conf.server'}, + tries = 1, user = 'hive') + + self.assertResourceCalled('Execute', 'hdp-select set hive-metastore %s' % version,) + + self.assertNoMoreResources() \ No newline at end of file
