Repository: ambari Updated Branches: refs/heads/trunk 6adee814f -> 2a253c056
AMBARI-12279 - Oozie Rolling Upgrade Fails Due To Missing Database JAR (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2a253c05 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2a253c05 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2a253c05 Branch: refs/heads/trunk Commit: 2a253c056c9718e64d69b622d405efa17e69fd27 Parents: 6adee81 Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Sat Jul 4 20:52:18 2015 -0400 Committer: Jonathan Hurley <jhur...@hortonworks.com> Committed: Mon Jul 6 16:10:12 2015 -0400 ---------------------------------------------------------------------- .../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 55 ++++++++++----- .../package/scripts/oozie_server_upgrade.py | 24 +++---- .../stacks/2.0.6/OOZIE/test_oozie_server.py | 70 ++++++++++++++++++++ .../stacks/2.2/configs/oozie-upgrade.json | 10 +-- 4 files changed, 123 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/2a253c05/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py index 8441f11..9b5177f 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py @@ -220,23 +220,8 @@ def oozie_server_specific(): sudo = True, ) - if params.jdbc_driver_name=="com.mysql.jdbc.Driver" or \ - params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \ - params.jdbc_driver_name=="oracle.jdbc.driver.OracleDriver": - File(params.downloaded_custom_connector, - content = DownloadSource(params.driver_curl_source), - ) - - - Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.target), - #creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc - path=["/bin", "/usr/bin/"], - sudo = True) - - File ( params.target, - owner = params.oozie_user, - group = params.user_group - ) + # download the database JAR + download_database_library_if_needed() #falcon el extension if params.has_falcon_host: @@ -289,3 +274,39 @@ def oozie_server_specific(): Execute(('chown', '-R', format("{oozie_user}:{user_group}"), params.oozie_server_dir), sudo=True ) + +def download_database_library_if_needed(target_directory = None): + """ + Downloads the library to use when connecting to the Oozie database, if + necessary. The library will be downloaded to 'params.target' unless + otherwise specified. + :param target_directory: the location where the database library will be + downloaded to. + :return: + """ + import params + jdbc_drivers = ["com.mysql.jdbc.Driver", + "com.microsoft.sqlserver.jdbc.SQLServerDriver", + "oracle.jdbc.driver.OracleDriver"] + + # check to see if the JDBC driver name is in the list of ones that need to + # be downloaded + if params.jdbc_driver_name not in jdbc_drivers: + return + + # if the target directory is not specified + if target_directory is None: + target_jar_with_directory = params.target + else: + # create the full path using the supplied target directory and the JDBC JAR + target_jar_with_directory = target_directory + os.path.sep + params.jdbc_driver_jar + + File(params.downloaded_custom_connector, + content = DownloadSource(params.driver_curl_source)) + + Execute(('cp', '--remove-destination', params.downloaded_custom_connector, target_jar_with_directory), + path=["/bin", "/usr/bin/"], + sudo = True) + + File(target_jar_with_directory, owner = params.oozie_user, + group = params.user_group) http://git-wip-us.apache.org/repos/asf/ambari/blob/2a253c05/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py index bd423a2..fb19ddf 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py @@ -33,8 +33,8 @@ from resource_management.libraries.functions import hdp_select from resource_management.libraries.functions import format_hdp_stack_version from resource_management.libraries.functions import tar_archive from resource_management.libraries.script.script import Script -from resource_management.core.resources import File -from resource_management.core.source import DownloadSource + +import oozie BACKUP_TEMP_DIR = "oozie-upgrade-backup" BACKUP_CONF_ARCHIVE = "oozie-conf-backup.tar" @@ -156,18 +156,7 @@ class OozieUpgrade(Script): shutil.copy2(oozie_ext_zip_file, params.oozie_libext_customer_dir) # Redownload jdbc driver to a new current location - if params.jdbc_driver_name=="com.mysql.jdbc.Driver" or \ - params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \ - params.jdbc_driver_name=="oracle.jdbc.driver.OracleDriver": - - File(params.downloaded_custom_connector, - content = DownloadSource(params.driver_curl_source)) - - Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.target), - #creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc - path=["/bin", "/usr/bin/"], sudo = True) - - File(params.target, owner = params.oozie_user, group = params.user_group) + oozie.download_database_library_if_needed() @staticmethod @@ -230,6 +219,13 @@ class OozieUpgrade(Script): # upgrade oozie DB Logger.info('Upgrading the Oozie database...') + + # the database upgrade requires the db driver JAR, but since we have + # not yet run hdp-select to upgrade the current points, we have to use + # the versioned libext directory as the location[[-vufdtffr, + versioned_libext_dir = "/usr/hdp/{0}/oozie/libext".format(stack_version) + oozie.download_database_library_if_needed(target_directory=versioned_libext_dir) + database_upgrade_command = "/usr/hdp/{0}/oozie/bin/ooziedb.sh upgrade -run".format(stack_version) Execute(database_upgrade_command, user=params.oozie_user, logoutput=True) http://git-wip-us.apache.org/repos/asf/ambari/blob/2a253c05/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py index 0ce0f20..0759678 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py @@ -1153,3 +1153,73 @@ class TestOozieServer(RMFTestCase): self.assertResourceCalled('Execute', '/usr/hdp/2.3.0.0-1234/oozie/bin/oozie-setup.sh sharelib create -fs hdfs://c6401.ambari.apache.org:8020', user='oozie', logoutput = True) + + def test_upgrade_database_sharelib_existing_mysql(self): + """ + Tests that the upgrade script runs the proper commands before the + actual upgrade begins when Oozie is using and external database. This + should ensure that the JDBC JAR is copied. + :return: + """ + config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/oozie-upgrade.json" + + with open(config_file, "r") as f: + json_content = json.load(f) + + version = '2.3.0.0-1234' + json_content['commandParams']['version'] = version + json_content['hostLevelParams']['stack_name'] = "HDP" + + # use mysql external database + json_content['configurations']['oozie-site']['oozie.service.JPAService.jdbc.driver'] = "com.mysql.jdbc.Driver" + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server_upgrade.py", + classname = "OozieUpgrade", command = "upgrade_oozie_database_and_sharelib", + config_dict = json_content, + hdp_stack_version = self.UPGRADE_STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES ) + + self.assertResourceCalled('File', '/tmp/mysql-connector-java.jar', + content = DownloadSource('http://c6401.ambari.apache.org:8080/resources//mysql-jdbc-driver.jar') ) + + self.assertResourceCalled('Execute', ('cp', '--remove-destination', '/tmp/mysql-connector-java.jar', + '/usr/hdp/2.3.0.0-1234/oozie/libext/mysql-connector-java.jar'), + path = ['/bin', '/usr/bin/'], sudo = True) + + self.assertResourceCalled('File', '/usr/hdp/2.3.0.0-1234/oozie/libext/mysql-connector-java.jar', + owner = 'oozie', group = 'hadoop' ) + + self.assertResourceCalled('Execute', '/usr/hdp/2.3.0.0-1234/oozie/bin/ooziedb.sh upgrade -run', + user = 'oozie', logoutput = True ) + + self.assertResourceCalled('HdfsResource', '/user/oozie/share', + security_enabled = False, + hadoop_bin_dir = '/usr/hdp/2.3.0.0-1234/hadoop/bin', + keytab = UnknownConfigurationMock(), + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + user = 'hdfs', + hdfs_site = UnknownConfigurationMock(), + kinit_path_local = '/usr/bin/kinit', + principal_name = UnknownConfigurationMock(), + recursive_chmod = True, + owner = 'oozie', + group = 'hadoop', + hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', + type = 'directory', + action = ['create_on_execute'], + mode = 0755 ) + + self.assertResourceCalled('HdfsResource', None, + security_enabled = False, + hadoop_bin_dir = '/usr/hdp/2.3.0.0-1234/hadoop/bin', + keytab = UnknownConfigurationMock(), + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + hdfs_site = UnknownConfigurationMock(), + kinit_path_local = '/usr/bin/kinit', + principal_name = UnknownConfigurationMock(), + user = 'hdfs', + action = ['execute'], + hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf' ) + + self.assertResourceCalled('Execute', '/usr/hdp/2.3.0.0-1234/oozie/bin/oozie-setup.sh sharelib create -fs hdfs://c6401.ambari.apache.org:8020', + user='oozie', logoutput = True) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/2a253c05/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json b/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json index 1f98793..a2e8b1d 100644 --- a/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json +++ b/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json @@ -31,14 +31,14 @@ "clusterName": "c1", "hostname": "c6402.ambari.apache.org", "hostLevelParams": { - "jdk_location": "http://hw10897.ix:8080/resources/";, + "jdk_location": "http://c6401.ambari.apache.org:8080/resources/";, "ambari_db_rca_password": "mapred", "java_home": "/usr/jdk64/jdk1.7.0_45", "java_version": "8", - "ambari_db_rca_url": "jdbc:postgresql://hw10897.ix/ambarirca", + "ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca", "stack_name": "HDP", "custom_command": "RESTART", - "oracle_jdbc_url": "http://hw10897.ix:8080/resources//ojdbc6.jar";, + "oracle_jdbc_url": "http://c6401.ambari.apache.org:8080/resources//ojdbc6.jar";, "repo_info": "[{\"baseUrl\":\"http://repo.ambari.apache.org/hdp/centos6/HDP-2.2.0.0/\",\"osType\":\"redhat6\",\"repoId\":\"HDP-2.2\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/GA/2.2.0.0\",\"latestBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/GA/2.2.0.0\"},{\"baseUrl\":\"http://repo.ambari.apache.org/hdp/centos6/HDP-UTILS-1.1.0.20/\",\"osType\":\"redhat6\",\"repoId\":\"HDP-UTILS-1.1.0.20\",\"repoName\":\"HDP-UTILS\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6\",\"latestBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6\"}]";, "group_list": "[\"hadoop\",\"nobody\",\"users\"]", "agentCacheDir": "/var/lib/ambari-agent/cache", @@ -49,7 +49,7 @@ "ambari_db_rca_username": "mapred", "db_driver_filename": "mysql-connector-java.jar", "user_list": "[\"nobody\",\"oozie\",\"hive\",\"mapred\",\"hbase\",\"ambari-qa\",\"zookeeper\",\"tez\",\"hdfs\",\"falcon\",\"yarn\",\"hcat\"]", - "mysql_jdbc_url": "http://hw10897.ix:8080/resources//mysql-connector-java.jar"; + "mysql_jdbc_url": "http://c6401.ambari.apache.org:8080/resources//mysql-connector-java.jar"; }, "commandType": "EXECUTION_COMMAND", "roleParams": { @@ -225,7 +225,7 @@ "8670" ], "ambari_server_host": [ - "hw10897.ix" + "c6401.ambari.apache.org" ], "rm_host": [ "c6402.ambari.apache.org"
