Repository: ambari Updated Branches: refs/heads/trunk cd60e5aac -> e8dde01e3
AMBARI-6008. Incorrect behaviour when trying to execute upgradestack to nonexistent HDP version (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e8dde01e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e8dde01e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e8dde01e Branch: refs/heads/trunk Commit: e8dde01e380e0328287de1ba528a022aff89ae8e Parents: cd60e5a Author: Andrew Onishuk <[email protected]> Authored: Fri Jun 6 17:41:00 2014 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Fri Jun 6 17:41:00 2014 +0300 ---------------------------------------------------------------------- .../server/upgrade/StackUpgradeHelper.java | 84 ++++++++++++++------ ambari-server/src/main/python/ambari-server.py | 32 +++++--- .../server/upgrade/StackUpgradeHelperTest.java | 51 ++++++++++++ .../src/test/python/TestAmbariServer.py | 49 ++++++++++-- 4 files changed, 174 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e8dde01e/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeHelper.java index 77ec069..6420ca7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeHelper.java @@ -24,8 +24,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import javax.ws.rs.core.UriInfo; - import org.apache.ambari.server.controller.ControllerModule; import org.apache.ambari.server.orm.DBAccessor; import org.apache.ambari.server.orm.dao.MetainfoDAO; @@ -39,10 +37,14 @@ import com.google.inject.Inject; import com.google.inject.Injector; import com.google.inject.persist.PersistService; import com.google.inject.persist.Transactional; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import org.eclipse.jetty.http.HttpStatus; public class StackUpgradeHelper { - private static final Logger LOG = LoggerFactory.getLogger - (StackUpgradeHelper.class); + + private static final Logger LOG = LoggerFactory.getLogger(StackUpgradeHelper.class); private static final String STACK_ID_UPDATE_ACTION = "updateStackId"; private static final String METAINFO_UPDATE_ACTION = "updateMetaInfo"; @@ -67,6 +69,7 @@ public class StackUpgradeHelper { /** * Add key value to the metainfo table. + * * @param data * @throws SQLException */ @@ -91,6 +94,7 @@ public class StackUpgradeHelper { /** * Change the stack id in the Ambari DB. + * * @param stackInfo * @throws SQLException */ @@ -98,44 +102,75 @@ public class StackUpgradeHelper { if (stackInfo == null || stackInfo.isEmpty()) { throw new IllegalArgumentException("Empty stack id. " + stackInfo); } - + String repoUrl = stackInfo.remove("repo_url"); String repoUrlOs = stackInfo.remove("repo_url_os"); - + Iterator<Map.Entry<String, String>> stackIdEntry = stackInfo.entrySet().iterator(); Map.Entry<String, String> stackEntry = stackIdEntry.next(); String stackName = stackEntry.getKey(); String stackVersion = stackEntry.getValue(); - LOG.info("Updating stack id, stackName = " + stackName + ", " + - "stackVersion = "+ stackVersion); - - stackUpgradeUtil.updateStackDetails(stackName, stackVersion); + LOG.info("Updating stack id, stackName = " + stackName + ", " + + "stackVersion = " + stackVersion); - if (null != repoUrl) { - stackUpgradeUtil.updateLocalRepo(stackName, stackVersion, repoUrl, repoUrlOs); + if (null != repoUrl && !repoUrl.isEmpty() && repoUrl.startsWith("http")) { + if (!repoUrl.trim().endsWith("/")) repoUrl = repoUrl + "/"; + if (checkURL(repoUrl + "repodata/repomd.xml", 2000)) { + stackUpgradeUtil.updateLocalRepo(stackName, stackVersion, repoUrl, repoUrlOs); + } + } else { + throw (new Exception("Base repo URL not found")); } - dbAccessor.updateTable("hostcomponentstate", "current_state", "INSTALLED", "where current_state = 'UPGRADING'"); + stackUpgradeUtil.updateStackDetails(stackName, stackVersion); + + dbAccessor.updateTable("hostcomponentstate", "current_state", + "INSTALLED", "where current_state = 'UPGRADING'"); + } + + public boolean checkURL(String url, int timeout) throws Exception { + int responseCode = 0; + try { + HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); + connection.setConnectTimeout(timeout); + connection.setReadTimeout(timeout); + connection.setRequestMethod("HEAD"); + responseCode = connection.getResponseCode(); + if (responseCode != HttpStatus.OK_200) { + throw (new Exception("Invalid repository base URL" + + ", Responce: "+ HttpStatus.getMessage(responseCode) + + ", during check URL: " + url)); + } else { + return true; + } + } catch (IOException exception) { + throw (new Exception(exception.getClass().getSimpleName() + + ", Responce: "+ HttpStatus.getMessage(responseCode) + + ", during check URL: " + url)); + } } private List<String> getValidActions() { - return new ArrayList<String>() {{ - add(STACK_ID_UPDATE_ACTION); - add(METAINFO_UPDATE_ACTION); - }}; + return new ArrayList<String>() { + { + add(STACK_ID_UPDATE_ACTION); + add(METAINFO_UPDATE_ACTION); + } + }; } /** * Support changes need to support upgrade of Stack + * * @param args Simple key value json map */ public static void main(String[] args) { try { if (args.length < 2) { - throw new InputMismatchException("Need to provide action, " + - "stack name and stack version."); + throw new InputMismatchException("Need to provide action, " + + "stack name and stack version."); } String action = args[0]; @@ -146,17 +181,16 @@ public class StackUpgradeHelper { Gson gson = injector.getInstance(Gson.class); if (!stackUpgradeHelper.getValidActions().contains(action)) { - throw new IllegalArgumentException("Unsupported action. Allowed " + - "actions: " + stackUpgradeHelper.getValidActions()); + throw new IllegalArgumentException("Unsupported action. Allowed " + + "actions: " + stackUpgradeHelper.getValidActions()); } - stackUpgradeHelper.startPersistenceService(); Map values = gson.fromJson(valueMap, Map.class); if (action.equals(STACK_ID_UPDATE_ACTION)) { stackUpgradeHelper.updateStackVersion(values); - + } else if (action.equals(METAINFO_UPDATE_ACTION)) { stackUpgradeHelper.updateMetaInfo(values); @@ -165,8 +199,8 @@ public class StackUpgradeHelper { stackUpgradeHelper.stopPersistenceService(); } catch (Throwable t) { - LOG.error("Caught exception on upgrade. Exiting...", t); - System.exit(1); + System.err.println("Caught exception on upgrade. Exiting..." + t.getMessage()); + System.exit(2); } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/e8dde01e/ambari-server/src/main/python/ambari-server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py index d580634..7841955 100755 --- a/ambari-server/src/main/python/ambari-server.py +++ b/ambari-server/src/main/python/ambari-server.py @@ -2605,13 +2605,26 @@ def upgrade_stack(args, stack_id, repo_url=None, repo_url_os=None): raise FatalException(4, err) check_database_name_property() + local_repo_check_commamd = "" + if OS_FAMILY == OSConst.DEBIAN_FAMILY: + raise FatalException(3, '"upgradestack" command not supported yet for Debian OS\'es family.') + elif OS_FAMILY == OSConst.REDHAT_FAMILY: + local_repo_check_commamd = 'yum repolist | grep "{0} "' + elif OS_FAMILY == OSConst.SUSE_FAMILY: + local_repo_check_commamd = 'zypper repos | grep "{0} "' + + command = local_repo_check_commamd.format(stack_id) + (retcode, stdout, stderr) = run_in_shell(command) + + if not retcode == 0 and repo_url is None: + raise FatalException(retcode, 'Repository for ' + stack_id + " is not existed") + stack_name, stack_version = stack_id.split(STACK_NAME_VER_SEP) - retcode = run_stack_upgrade(stack_name, stack_version, repo_url, repo_url_os) + retcode, stdout, stderr = run_stack_upgrade(stack_name, stack_version, repo_url, repo_url_os) if not retcode == 0: - raise FatalException(retcode, 'Stack upgrade failed.') + raise FatalException(retcode, 'Error executing stack upgrade. ' + stderr) - return retcode def load_stack_values(version, filename): @@ -2723,10 +2736,9 @@ def run_schema_upgrade(): def run_stack_upgrade(stackName, stackVersion, repo_url, repo_url_os): jdk_path = find_jdk() if jdk_path is None: - print_error_msg("No JDK found, please run the \"setup\" " - "command to install a JDK automatically or install any " - "JDK manually to " + JDK_INSTALL_DIR) - return 1 + return 1, "", "No JDK found, please run the \"setup\" \ + command to install a JDK automatically or install any \ + JDK manually to " + JDK_INSTALL_DIR stackId = {} stackId[stackName] = stackVersion if repo_url is not None: @@ -2737,11 +2749,7 @@ def run_stack_upgrade(stackName, stackVersion, repo_url, repo_url_os): command = STACK_UPGRADE_HELPER_CMD.format(jdk_path, get_conf_dir(), get_ambari_classpath(), "updateStackId", "'" + json.dumps(stackId) + "'") - (retcode, stdout, stderr) = run_os_command(command) - print_info_msg("Return code from stack upgrade command, retcode = " + str(retcode)) - if retcode > 0: - print_error_msg("Error executing stack upgrade, please check the server logs.") - return retcode + return run_os_command(command) def run_metainfo_upgrade(keyValueMap=None): http://git-wip-us.apache.org/repos/asf/ambari/blob/e8dde01e/ambari-server/src/test/java/org/apache/ambari/server/upgrade/StackUpgradeHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/StackUpgradeHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/StackUpgradeHelperTest.java new file mode 100644 index 0000000..1a98447 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/StackUpgradeHelperTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.upgrade; + +import java.util.HashMap; +import java.util.Map; +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * + * @author root + */ +public class StackUpgradeHelperTest { + + public StackUpgradeHelperTest() { + } + + @Test + public void testUpdateStackVersion() { + System.out.println("updateStackVersion"); + Map<String, String> stackInfo = new HashMap<String, String>(); + stackInfo.put("repo_url", "http://foo.bar";); + stackInfo.put("repo_url_os", "centos6"); + stackInfo.put("HDP", "1.3.0"); + StackUpgradeHelper instance = new StackUpgradeHelper(); + try { + instance.updateStackVersion(stackInfo); + } catch (Exception ex) { + assertEquals("UnknownHostException, Responce: 0, " + + "during check URL: http://foo.bar/repodata/repomd.xml";, ex.getMessage()); + } + } + + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/e8dde01e/ambari-server/src/test/python/TestAmbariServer.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py index 0ed238a..27fa276 100644 --- a/ambari-server/src/test/python/TestAmbariServer.py +++ b/ambari-server/src/test/python/TestAmbariServer.py @@ -1091,7 +1091,7 @@ class TestAmbariServer(TestCase): # Expected self.assertTrue("JCE Policy path" in fe.reason) pass - os_path_exists_mock.reset() + os_path_exists_mock.reset_mock() # Case when JCE is a directory os_path_exists_mock.return_value = True @@ -1103,7 +1103,7 @@ class TestAmbariServer(TestCase): # Expected self.assertTrue("JCE Policy path is a directory" in fe.reason) pass - os_path_isdir_mock.reset() + os_path_isdir_mock.reset_mock() os_path_isdir_mock.return_value = False os_path_join_mock.return_value = \ @@ -2783,11 +2783,12 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV self.assertTrue(removeMock.called) + @patch.object(ambari_server, "run_in_shell") @patch.object(ambari_server, "is_root") @patch.object(ambari_server, "check_database_name_property") @patch.object(ambari_server, "run_stack_upgrade") def test_upgrade_stack(self, run_stack_upgrade_mock, - check_database_name_property_mock, is_root_mock): + check_database_name_property_mock, is_root_mock, run_in_shell_mock): args = MagicMock() args.persistence_type = "local" @@ -2802,12 +2803,50 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV pass # Testing calls under root + run_in_shell_mock.return_value = 0, "existed!", "" is_root_mock.return_value = True - run_stack_upgrade_mock.return_value = 0 + run_stack_upgrade_mock.return_value = 0, "", "" ambari_server.upgrade_stack(args, 'HDP-2.0') self.assertTrue(run_stack_upgrade_mock.called) run_stack_upgrade_mock.assert_called_with("HDP", "2.0", None, None) + run_stack_upgrade_mock.reset_mock() + + # Testing calls if desired stack repo not existed + run_in_shell_mock.return_value = 1, "", "" + is_root_mock.return_value = True + try: + ambari_server.upgrade_stack(args, 'HDP-2.0') + self.fail("Should throw exception") + except FatalException as fe: + # Expected + self.assertTrue("Repository for HDP-2.0 is not existed" in fe.reason) + pass + + run_stack_upgrade_mock.reset_mock() + + # Testing calls if desired stack repo not existed but base URL is not empty + run_in_shell_mock.return_value = 0, "existed!", "" + is_root_mock.return_value = True + run_stack_upgrade_mock.return_value = 0, "", "" + ambari_server.upgrade_stack(args, 'HDP-2.0', "URL") + + self.assertTrue(run_stack_upgrade_mock.called) + run_stack_upgrade_mock.assert_called_with("HDP", "2.0", "URL", None) + run_stack_upgrade_mock.reset_mock() + + # Testing calls if upgrade stack return non zero retcode + run_in_shell_mock.return_value = 0, "", "" + is_root_mock.return_value = True + run_stack_upgrade_mock.return_value = 2, "", "" + try: + ambari_server.upgrade_stack(args, 'HDP-2.0', "URL") + self.fail("Should throw exception") + except FatalException as fe: + # Expected + self.assertTrue("Error executing stack upgrade." in fe.reason) + pass + @patch.object(ambari_server, 'get_conf_dir') @patch.object(ambari_server, 'get_ambari_classpath') @@ -3734,7 +3773,7 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV copy_files_mock.assert_called_with(drivers_list, resources_dir) # Non-Silent option, no drivers at first ask, present drivers after that - find_jdbc_driver_mock.reset() + find_jdbc_driver_mock.reset_mock() find_jdbc_driver_mock.side_effect = [-1, -1] rcode = ambari_server.check_jdbc_drivers(args)
