AMBARI-13518 Host cleanup should remove only stack directories from /usr/hdp (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a8445c9a Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a8445c9a Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a8445c9a Branch: refs/heads/branch-dev-patch-upgrade Commit: a8445c9aa699622be486695fcfc6a1c9cf10a518 Parents: b9bb0d3 Author: Dmytro Sen <d...@apache.org> Authored: Thu Oct 22 17:18:18 2015 +0300 Committer: Dmytro Sen <d...@apache.org> Committed: Thu Oct 22 17:18:18 2015 +0300 ---------------------------------------------------------------------- .../ambari_agent/HostCheckReportFileHandler.py | 44 ++++++++++-- .../TestHostCheckReportFileHandler.py | 72 ++++++++++++++------ 2 files changed, 89 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a8445c9a/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py b/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py index 794e427..1f87a73 100644 --- a/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py +++ b/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -''' +""" Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information @@ -16,15 +16,19 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -''' +""" import datetime import os.path import logging +import re import traceback from AmbariConfig import AmbariConfig -import ConfigParser; +import ConfigParser +HADOOP_ROOT_DIR = "/usr/hdp" +HADOOP_PERM_REMOVE_LIST = ["current"] +HADOOP_ITEMDIR_REGEX = "(\d\.){3}\d-\d{4}" logger = logging.getLogger(__name__) class HostCheckReportFileHandler: @@ -83,6 +87,37 @@ class HostCheckReportFileHandler: logger.error("Can't write host check file at %s :%s " % (self.hostCheckCustomActionsFilePath, err.message)) traceback.print_exc() + def _hdp_list_directory(self): + """ + Return filtered list of /usr/hdp directory allowed to be removed + :rtype list + """ + + if not os.path.exists(HADOOP_ROOT_DIR): + return [] + + matcher = re.compile(HADOOP_ITEMDIR_REGEX) # pre-compile regexp + folder_content = os.listdir(HADOOP_ROOT_DIR) + remove_list = [] + + remlist_items_count = 0 + + for item in folder_content: + full_path = "%s%s%s" % (HADOOP_ROOT_DIR, os.path.sep, item) + if item in HADOOP_PERM_REMOVE_LIST: + remove_list.append(full_path) + remlist_items_count += 1 + + if matcher.match(item) is not None: + remove_list.append(full_path) + remlist_items_count += 1 + + # if remove list have same length as target folder, assume that they are identical + if remlist_items_count == len(folder_content): + remove_list.append(HADOOP_ROOT_DIR) + + return remove_list + def writeHostCheckFile(self, hostInfo): if self.hostCheckFilePath is None: return @@ -117,8 +152,7 @@ class HostCheckReportFileHandler: items = [] for itemDetail in hostInfo['stackFoldersAndFiles']: items.append(itemDetail['name']) - if os.path.exists('/usr/hdp'): - items.append('/usr/hdp') + items += self._hdp_list_directory() config.add_section('directories') config.set('directories', 'dir_list', ','.join(items)) http://git-wip-us.apache.org/repos/asf/ambari/blob/a8445c9a/ambari-agent/src/test/python/ambari_agent/TestHostCheckReportFileHandler.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostCheckReportFileHandler.py b/ambari-agent/src/test/python/ambari_agent/TestHostCheckReportFileHandler.py index d56ad8f..c595082 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestHostCheckReportFileHandler.py +++ b/ambari-agent/src/test/python/ambari_agent/TestHostCheckReportFileHandler.py @@ -20,6 +20,7 @@ limitations under the License. from unittest import TestCase import unittest +from mock.mock import patch import os import tempfile from ambari_agent.HostCheckReportFileHandler import HostCheckReportFileHandler @@ -38,8 +39,8 @@ class TestHostCheckReportFileHandler(TestCase): config.set('agent', 'prefix', os.path.dirname(tmpfile)) handler = HostCheckReportFileHandler(config) - dict = {} - handler.writeHostCheckFile(dict) + mydict = {} + handler.writeHostCheckFile(mydict) configValidator = ConfigParser.RawConfigParser() configPath = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_FILE) @@ -56,16 +57,16 @@ class TestHostCheckReportFileHandler(TestCase): config.set('agent', 'prefix', os.path.dirname(tmpfile)) handler = HostCheckReportFileHandler(config) - dict = {} - dict['hostHealth'] = {} - dict['existingUsers'] = [] - dict['alternatives'] = [] - dict['stackFoldersAndFiles'] = [] - dict['hostHealth']['activeJavaProcs'] = [] - dict['installedPackages'] = [] - dict['existingRepos'] = [] + mydict = {} + mydict['hostHealth'] = {} + mydict['existingUsers'] = [] + mydict['alternatives'] = [] + mydict['stackFoldersAndFiles'] = [] + mydict['hostHealth']['activeJavaProcs'] = [] + mydict['installedPackages'] = [] + mydict['existingRepos'] = [] - handler.writeHostCheckFile(dict) + handler.writeHostCheckFile(mydict) configValidator = ConfigParser.RawConfigParser() configPath = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_FILE) @@ -96,19 +97,19 @@ class TestHostCheckReportFileHandler(TestCase): handler = HostCheckReportFileHandler(config) - dict = {} - dict['hostHealth'] = {} - dict['existingUsers'] = [{'name':'user1', 'homeDir':'/var/log', 'status':'Exists'}] - dict['alternatives'] = [ + mydict = {} + mydict['hostHealth'] = {} + mydict['existingUsers'] = [{'name':'user1', 'homeDir':'/var/log', 'status':'Exists'}] + mydict['alternatives'] = [ {'name':'/etc/alternatives/hadoop-conf', 'target':'/etc/hadoop/conf.dist'}, {'name':'/etc/alternatives/hbase-conf', 'target':'/etc/hbase/conf.1'} ] - dict['stackFoldersAndFiles'] = [{'name':'/a/b', 'type':'directory'},{'name':'/a/b.txt', 'type':'file'}] - dict['hostHealth']['activeJavaProcs'] = [ + mydict['stackFoldersAndFiles'] = [{'name':'/a/b', 'type':'directory'},{'name':'/a/b.txt', 'type':'file'}] + mydict['hostHealth']['activeJavaProcs'] = [ {'pid':355,'hadoop':True,'command':'some command','user':'root'}, {'pid':455,'hadoop':True,'command':'some command','user':'hdfs'} ] - handler.writeHostCheckFile(dict) + handler.writeHostCheckFile(mydict) configValidator = ConfigParser.RawConfigParser() configPath = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_FILE) @@ -130,13 +131,13 @@ class TestHostCheckReportFileHandler(TestCase): self.chkItemsEqual(procs, ['455', '355']) - dict['installed_packages'] = [ + mydict['installed_packages'] = [ {'name':'hadoop','version':'3.2.3','repoName':'HDP'}, {'name':'hadoop-lib','version':'3.2.3','repoName':'HDP'} ] - dict['existing_repos'] = ['HDP', 'HDP-epel'] + mydict['existing_repos'] = ['HDP', 'HDP-epel'] - handler.writeHostChecksCustomActionsFile(dict) + handler.writeHostChecksCustomActionsFile(mydict) configValidator = ConfigParser.RawConfigParser() configPath_ca = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_CUSTOM_ACTIONS_FILE) configValidator.read(configPath_ca) @@ -150,13 +151,40 @@ class TestHostCheckReportFileHandler(TestCase): time = configValidator.get('metadata', 'created') self.assertTrue(time != None) + @patch("os.path.exists") + @patch("os.listdir") + def test_write_host_stack_list(self, list_mock, exists_mock): + exists_mock.return_value = True + list_mock.return_value = ["1.1.1.1-1234", "current", "test"] + + tmpfile = tempfile.mktemp() + + config = ConfigParser.RawConfigParser() + config.add_section('agent') + config.set('agent', 'prefix', os.path.dirname(tmpfile)) + + handler = HostCheckReportFileHandler(config) + + mydict = {} + mydict['hostHealth'] = {} + mydict['stackFoldersAndFiles'] = [{'name':'/a/b', 'type':'directory'},{'name':'/a/b.txt', 'type':'file'}] + + handler.writeHostCheckFile(mydict) + + configValidator = ConfigParser.RawConfigParser() + configPath = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_FILE) + configValidator.read(configPath) + + paths = configValidator.get('directories', 'dir_list') + self.chkItemsEqual(paths, ['/a/b', '/a/b.txt', '/usr/hdp/1.1.1.1-1234', '/usr/hdp/current']) + def chkItemsEqual(self, commaDelimited, items): items1 = commaDelimited.split(',') items1.sort() items.sort() items1Str = ','.join(items1) items2Str = ','.join(items) - self.assertEquals(items1Str, items2Str) + self.assertEquals(items2Str, items1Str) if __name__ == "__main__": unittest.main(verbosity=2)
