Repository: ambari Updated Branches: refs/heads/trunk dfe66f690 -> 5e093c9eb
AMBARI-15348 Ambari upgrade/start commands should detect any ambari-server hotfix and fail upgrade/start from proceeding (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5e093c9e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5e093c9e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5e093c9e Branch: refs/heads/trunk Commit: 5e093c9ebc120dbdb86356b100b49f6da7c05747 Parents: dfe66f6 Author: Dmytro Sen <d...@apache.org> Authored: Tue Mar 15 17:58:04 2016 +0200 Committer: Dmytro Sen <d...@apache.org> Committed: Tue Mar 15 17:58:04 2016 +0200 ---------------------------------------------------------------------- ambari-server/pom.xml | 4 -- .../src/main/package/deb/control/preinst | 14 ++++ .../src/main/package/rpm/preinstall.sh | 14 ++++ .../python/ambari_server/serverClassPath.py | 68 ++++++++++++++---- .../main/python/ambari_server/serverUpgrade.py | 2 +- .../src/main/python/ambari_server_main.py | 4 +- .../src/test/python/TestServerClassPath.py | 76 +++++++++++++++++++- 7 files changed, 162 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/5e093c9e/ambari-server/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml index 83424c2..5bb3b98 100644 --- a/ambari-server/pom.xml +++ b/ambari-server/pom.xml @@ -1006,10 +1006,6 @@ </dependency> <dependency> <groupId>org.apache.ant</groupId> - <artifactId>ant</artifactId> - </dependency> - <dependency> - <groupId>org.apache.ant</groupId> <artifactId>ant-launcher</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/ambari/blob/5e093c9e/ambari-server/src/main/package/deb/control/preinst ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/package/deb/control/preinst b/ambari-server/src/main/package/deb/control/preinst index eb94f95..34af1c8 100644 --- a/ambari-server/src/main/package/deb/control/preinst +++ b/ambari-server/src/main/package/deb/control/preinst @@ -34,6 +34,8 @@ AMBARI_KRB_JAAS_LOGIN_FILE_OLD="$AMBARI_KRB_JAAS_LOGIN_FILE.rpmsave" AMBARI_VIEWS_FOLDER="${ROOT}/var/lib/ambari-server/resources/views" AMBARI_VIEWS_BACKUP_FOLDER="$AMBARI_VIEWS_FOLDER/backups" +AMBARI_SERVER_JAR_FILES="/usr/lib/ambari-server/ambari-server-*.jar" +AMBARI_SERVER_JAR_FILES_BACKUP_FOLDER="/usr/lib/ambari-server-backups" SERVER_CONF_SAVE="${ROOT}/etc/ambari-server/conf.save" SERVER_CONF_SAVE_BACKUP="${ROOT}/etc/ambari-server/conf_$(date '+%d_%m_%y_%H_%M').save" @@ -77,4 +79,16 @@ then cp -u $AMBARI_VIEWS_FOLDER/*.jar $AMBARI_VIEWS_BACKUP_FOLDER/ fi +for f in $AMBARI_SERVER_JAR_FILES; +do + if [ -f "$f" ] + then + if [ ! -d "$AMBARI_SERVER_JAR_FILES_BACKUP_FOLDER" ] + then + mkdir -p "$AMBARI_SERVER_JAR_FILES_BACKUP_FOLDER" + fi + mv -f $f $AMBARI_SERVER_JAR_FILES_BACKUP_FOLDER/ + fi +done + exit 0 http://git-wip-us.apache.org/repos/asf/ambari/blob/5e093c9e/ambari-server/src/main/package/rpm/preinstall.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/package/rpm/preinstall.sh b/ambari-server/src/main/package/rpm/preinstall.sh index a8e781e..6ed7444 100644 --- a/ambari-server/src/main/package/rpm/preinstall.sh +++ b/ambari-server/src/main/package/rpm/preinstall.sh @@ -24,6 +24,8 @@ COMMON_SERVICES_FOLDER_OLD="${ROOT}/var/lib/ambari-server/resources/common-servi AMBARI_VIEWS_FOLDER="${ROOT}/var/lib/ambari-server/resources/views" AMBARI_VIEWS_BACKUP_FOLDER="$AMBARI_VIEWS_FOLDER/backups" +AMBARI_SERVER_JAR_FILES="/usr/lib/ambari-server/ambari-server-*.jar" +AMBARI_SERVER_JAR_FILES_BACKUP_FOLDER="/usr/lib/ambari-server-backups" SERVER_CONF_SAVE="${ROOT}/etc/ambari-server/conf.save" SERVER_CONF_SAVE_BACKUP="${ROOT}/etc/ambari-server/conf_$(date '+%d_%m_%y_%H_%M').save" @@ -52,4 +54,16 @@ then cp -u $AMBARI_VIEWS_FOLDER/*.jar $AMBARI_VIEWS_BACKUP_FOLDER/ fi +for f in $AMBARI_SERVER_JAR_FILES; +do + if [ -f "$f" ] + then + if [ ! -d "$AMBARI_SERVER_JAR_FILES_BACKUP_FOLDER" ] + then + mkdir -p "$AMBARI_SERVER_JAR_FILES_BACKUP_FOLDER" + fi + mv -f $f $AMBARI_SERVER_JAR_FILES_BACKUP_FOLDER/ + fi +done + exit 0 http://git-wip-us.apache.org/repos/asf/ambari/blob/5e093c9e/ambari-server/src/main/python/ambari_server/serverClassPath.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverClassPath.py b/ambari-server/src/main/python/ambari_server/serverClassPath.py index d3ade3e..fdc1c12 100644 --- a/ambari-server/src/main/python/ambari_server/serverClassPath.py +++ b/ambari-server/src/main/python/ambari_server/serverClassPath.py @@ -17,26 +17,22 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ''' - -import datetime import glob + import os -import re -import shutil -import stat -import string -import sys -import tempfile import ambari_server -from ambari_commons.logging_utils import print_info_msg +import re +from ambari_commons.exceptions import FatalException +from ambari_commons.logging_utils import print_info_msg, print_warning_msg from resource_management.core.shell import quote_bash_args AMBARI_CONF_VAR = "AMBARI_CONF_DIR" SERVER_CLASSPATH_KEY = "SERVER_CLASSPATH" LIBRARY_PATH_KEY = "LD_LIBRARY_PATH" AMBARI_SERVER_LIB = "AMBARI_SERVER_LIB" JDBC_DRIVER_PATH_PROPERTY = "server.jdbc.driver.path" - +JAR_FILE_PATTERN = re.compile(r'^(.*)(-\d.*\.jar$)') +AMBARI_SERVER_JAR_FILE_PATTERN = re.compile(r'^ambari-server(-\d.*\.jar$)') class ServerClassPath(): @@ -92,9 +88,10 @@ class ServerClassPath(): return ambari_class_path - def get_full_ambari_classpath_escaped_for_shell(self): + def get_full_ambari_classpath_escaped_for_shell(self, validate_classpath=False): class_path = self._get_ambari_classpath() - + if validate_classpath: + self._validate_classpath(class_path) # When classpath is required we should also set native libs os env variable # This is required for some jdbc (ex. sqlAnywhere) self.set_native_libs_path() @@ -113,3 +110,50 @@ class ServerClassPath(): native_libs_path = os.environ[LIBRARY_PATH_KEY] + os.pathsep + native_libs_path os.environ[LIBRARY_PATH_KEY] = native_libs_path + def _validate_classpath(self, classpath): + """ + Check if java class path contains multiple versions of the same jar + archives, if yes - warn user. + If multiple versions of the ambari-server jar found, raise FatalException() + + Keyword arguments: + classpath - java class path + """ + jars = self._find_all_jars(classpath) + jar_names = {} + + for jar in jars: + match = JAR_FILE_PATTERN.match(os.path.basename(jar)) + if match: + for group in match.groups(): + if group in jar_names: + err = "Multiple versions of {0}.jar found in java class path " \ + "({1} and {2}). \n Make sure that you include only one " \ + "{0}.jar in the java class path '{3}'."\ + .format(group, jar, jar_names[group], classpath) + if AMBARI_SERVER_JAR_FILE_PATTERN.match(os.path.basename(jar)): + raise FatalException(1, err) + else: + print_warning_msg(err) + else: + jar_names[group] = jar + break + + def _find_all_jars(self, classpath): + """ + Return the list of absolute paths to jars in classpath. + Raise FatalException() if classpath isn't set + + Keyword arguments: + classpath - java class path + """ + if classpath: + jars = [] + for wildcard in classpath.split(os.pathsep): + for path in glob.glob(wildcard): + if os.path.isfile(path) and path.endswith(".jar"): + jars.append(path) + return jars + else: + raise FatalException(1, "No classpath specified.") + pass http://git-wip-us.apache.org/repos/asf/ambari/blob/5e093c9e/ambari-server/src/main/python/ambari_server/serverUpgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverUpgrade.py b/ambari-server/src/main/python/ambari_server/serverUpgrade.py index b51a623..9521d61 100644 --- a/ambari-server/src/main/python/ambari_server/serverUpgrade.py +++ b/ambari-server/src/main/python/ambari_server/serverUpgrade.py @@ -252,7 +252,7 @@ def run_schema_upgrade(args): print 'Upgrading database schema' serverClassPath = ServerClassPath(get_ambari_properties(), args) - class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell() + class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell(validate_classpath=True) command = SCHEMA_UPGRADE_HELPER_CMD.format(jdk_path, class_path) http://git-wip-us.apache.org/repos/asf/ambari/blob/5e093c9e/ambari-server/src/main/python/ambari_server_main.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server_main.py b/ambari-server/src/main/python/ambari_server_main.py index 8be3239..00e084b 100644 --- a/ambari-server/src/main/python/ambari_server_main.py +++ b/ambari-server/src/main/python/ambari_server_main.py @@ -286,8 +286,8 @@ def server_process_main(options, scmStatus=None): jvm_args += " -DskipDatabaseConsistencyValidation" param_list = generate_child_process_param_list(ambari_user, java_exe, - serverClassPath.get_full_ambari_classpath_escaped_for_shell(), debug_start, - suspend_mode) + serverClassPath.get_full_ambari_classpath_escaped_for_shell(validate_classpath=True), + debug_start, suspend_mode) environ = generate_env(options, ambari_user, current_user) if not os.path.exists(configDefaults.PID_DIR): http://git-wip-us.apache.org/repos/asf/ambari/blob/5e093c9e/ambari-server/src/test/python/TestServerClassPath.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestServerClassPath.py b/ambari-server/src/test/python/TestServerClassPath.py index 7ec36da..b843df6 100644 --- a/ambari-server/src/test/python/TestServerClassPath.py +++ b/ambari-server/src/test/python/TestServerClassPath.py @@ -20,6 +20,9 @@ import os os.environ["ROOT"] = "" import os +import shutil +import tempfile +from ambari_commons.exceptions import FatalException from mock.mock import patch, MagicMock from unittest import TestCase from ambari_server.properties import Properties @@ -29,7 +32,6 @@ from ambari_server.serverClassPath import ServerClassPath, AMBARI_SERVER_LIB, SE class TestConfigs(TestCase): - @patch("ambari_server.serverConfiguration.get_conf_dir") def test_server_class_path_default(self, get_conf_dir_mock): properties = Properties() @@ -90,3 +92,75 @@ class TestConfigs(TestCase): actual_classpath = serverClassPath.get_full_ambari_classpath_escaped_for_shell() self.assertEquals(expected_classpath, actual_classpath) + @patch("ambari_server.serverConfiguration.get_conf_dir") + def test_server_class_path_find_all_jars(self, get_conf_dir_mock): + temp_dir = tempfile.mkdtemp() + sub_dir = tempfile.mkdtemp(dir=temp_dir) + serverClassPath = ServerClassPath(None, None) + jar0 = tempfile.NamedTemporaryFile(suffix='.jar') + jar1 = tempfile.NamedTemporaryFile(suffix='.jar', dir=temp_dir) + jar2 = tempfile.NamedTemporaryFile(suffix='.jar', dir=temp_dir) + jar3 = tempfile.NamedTemporaryFile(suffix='.jar', dir=sub_dir) + # test /dir/*:file.jar + classpath = str(temp_dir) + os.path.sep + "*" + os.path.pathsep + jar0.name + jars = serverClassPath._find_all_jars(classpath) + self.assertEqual(len(jars), 3) + self.assertTrue(jar0.name in jars) + self.assertTrue(jar1.name in jars) + self.assertTrue(jar2.name in jars) + self.assertFalse(jar3.name in jars) + + # test no classpath specified + try: + serverClassPath._find_all_jars(None) + self.fail() + except FatalException as fe: + pass + + shutil.rmtree(temp_dir) + + @patch.object(ServerClassPath, "_find_all_jars") + @patch("ambari_server.serverConfiguration.get_conf_dir") + def test_server_class_path_validate_classpath(self, get_conf_dir_mock, + find_jars_mock): + serverClassPath = ServerClassPath(None, None) + + # No jars + find_jars_mock.return_value = [] + try: + serverClassPath._validate_classpath(None) + except: + self.fail() + + # Correct jars list + find_jars_mock.return_value = ["ambari-metrics-common-2.1.1.236.jar", + "ambari-server-2.1.1.236.jar", + "jetty-client-8.1.17.v20150415.jar", + "spring-core-3.0.7.RELEASE.jar"] + try: + serverClassPath._validate_classpath(None) + except: + self.fail() + + # Incorrect jars list, multiple versions for ambari-server.jar + find_jars_mock.return_value = ["ambari-metrics-common-2.1.1.236.jar", + "ambari-server-2.1.1.236.jar", + "ambari-server-2.1.1.hotfixed.jar", + "jetty-client-8.1.17.v20150415.jar", + "spring-core-3.0.7.RELEASE.jar"] + try: + serverClassPath._validate_classpath(None) + self.fail() + except: + pass + + # Incorrect jars list, multiple versions for not ambari-server.jar + find_jars_mock.return_value = ["ambari-metrics-common-2.1.1.236.jar", + "ambari-server-2.1.1.236.jar", + "jetty-client-8.1.17.v20150415.jar", + "jetty-client-9.jar", + "spring-core-3.0.7.RELEASE.jar"] + try: + serverClassPath._validate_classpath(None) + except: + self.fail()