Repository: ambari Updated Branches: refs/heads/branch-2.5 f8497d2c2 -> 2d13f6a05 refs/heads/trunk 3bab2125f -> fc36391cf
AMBARI-20596. Cleanup temporary files needed for downloading client configurations response (Attila Magyar via adoroszlai) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fc36391c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fc36391c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fc36391c Branch: refs/heads/trunk Commit: fc36391cf6e2b6aec04476af46eceaf91ceed01b Parents: 3bab212 Author: Attila Magyar <amag...@hortonworks.com> Authored: Mon Apr 3 12:05:16 2017 +0200 Committer: Attila Doroszlai <adorosz...@hortonworks.com> Committed: Mon Apr 3 12:05:16 2017 +0200 ---------------------------------------------------------------------- .../libraries/script/script.py | 8 +++-- .../internal/ClientConfigResourceProvider.java | 35 +++++++++++++++----- .../ClientConfigResourceProviderTest.java | 11 +++--- .../stacks/2.0.6/HDFS/test_hdfs_client.py | 11 +++--- 4 files changed, 44 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/fc36391c/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index fad14fd..5fa9ec4 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -874,6 +874,7 @@ class Script(object): Directory(self.get_tmp_dir(), create_parents = True) conf_tmp_dir = tempfile.mkdtemp(dir=self.get_tmp_dir()) + os.chmod(conf_tmp_dir, 0700) output_filename = os.path.join(self.get_tmp_dir(), config['commandParams']['output_file']) try: @@ -881,22 +882,23 @@ class Script(object): for filename, dict in file_dict.iteritems(): XmlConfig(filename, conf_dir=conf_tmp_dir, - mode=0644, + mode=0600, **self.generate_configs_get_xml_file_content(filename, dict) ) for file_dict in env_configs_list: for filename,dicts in file_dict.iteritems(): File(os.path.join(conf_tmp_dir, filename), - mode=0644, + mode=0600, content=InlineTemplate(self.generate_configs_get_template_file_content(filename, dicts))) for file_dict in properties_configs_list: for filename, dict in file_dict.iteritems(): PropertiesFile(os.path.join(conf_tmp_dir, filename), - mode=0644, + mode=0600, properties=self.generate_configs_get_xml_file_dict(filename, dict) ) with closing(tarfile.open(output_filename, "w:gz")) as tar: + os.chmod(output_filename, 0600) try: tar.add(conf_tmp_dir, arcname=os.path.basename(".")) finally: http://git-wip-us.apache.org/repos/asf/ambari/blob/fc36391c/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java index e98c062..e42bd45 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java @@ -213,6 +213,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv String TMP_PATH = configMap.get(Configuration.SERVER_TMP_DIR.getKey()); String pythonCmd = configMap.get(Configuration.AMBARI_PYTHON_WRAP.getKey()); List<String> pythonCompressFilesCmds = new ArrayList<>(); + List<File> commandFiles = new ArrayList<>(); for (ServiceComponentHostResponse response : componentMap.values()){ @@ -444,8 +445,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv jsonContent.put("clusterName", cluster.getClusterName()); jsonConfigurations = gson.toJson(jsonContent); - File jsonFileName = new File(TMP_PATH + File.separator + componentName + "-configuration.json"); - File tmpDirectory = new File(jsonFileName.getParent()); + File tmpDirectory = new File(TMP_PATH); if (!tmpDirectory.exists()) { try { tmpDirectory.mkdirs(); @@ -455,22 +455,33 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv throw new SystemException("Failed to get temporary directory to store configurations", se); } } + File jsonFile = File.createTempFile(componentName, "-configuration.json", tmpDirectory); + try { + jsonFile.setWritable(true, true); + jsonFile.setReadable(true, true); + } catch (SecurityException e) { + throw new SystemException("Failed to set permission", e); + } + PrintWriter printWriter = null; try { - printWriter = new PrintWriter(jsonFileName.getAbsolutePath()); + printWriter = new PrintWriter(jsonFile.getAbsolutePath()); printWriter.print(jsonConfigurations); printWriter.close(); } catch (FileNotFoundException e) { throw new SystemException("Failed to write configurations to json file ", e); } - String cmd = pythonCmd + " " + commandScriptAbsolute + " generate_configs " + jsonFileName.getAbsolutePath() + " " + + String cmd = pythonCmd + " " + commandScriptAbsolute + " generate_configs " + jsonFile.getAbsolutePath() + " " + packageFolderAbsolute + " " + TMP_PATH + File.separator + "structured-out.json" + " INFO " + TMP_PATH; + commandFiles.add(jsonFile); pythonCompressFilesCmds.add(cmd); } catch (AmbariException e) { throw new SystemException("Controller error ", e); + } catch (IOException e) { + throw new SystemException("Controller error ", e); } } @@ -483,11 +494,17 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv ExecutorService processExecutor = Executors.newFixedThreadPool(threadPoolSize); // put all threads that starts process to compress each component config files in the executor - List<CommandLineThreadWrapper> pythonCmdThreads = executeCommands(processExecutor, pythonCompressFilesCmds); - - // wait for all threads to finish - Integer timeout = configs.getExternalScriptTimeout(); - waitForAllThreadsToJoin(processExecutor, pythonCmdThreads, timeout); + try { + List<CommandLineThreadWrapper> pythonCmdThreads = executeCommands(processExecutor, pythonCompressFilesCmds); + + // wait for all threads to finish + Integer timeout = configs.getExternalScriptTimeout(); + waitForAllThreadsToJoin(processExecutor, pythonCmdThreads, timeout); + } finally { + for (File each : commandFiles) { + each.delete(); + } + } if (StringUtils.isEmpty(requestComponentName)) { TarUtils tarUtils; http://git-wip-us.apache.org/repos/asf/ambari/blob/fc36391c/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java index 8f0c66f..c2ee4d6 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java @@ -19,6 +19,7 @@ package org.apache.ambari.server.controller.internal; import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.anyString; import static org.easymock.EasyMock.createMock; import static org.easymock.EasyMock.createNiceMock; import static org.easymock.EasyMock.expect; @@ -26,7 +27,6 @@ import static org.easymock.EasyMock.expectLastCall; import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.verify; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; import static org.powermock.api.mockito.PowerMockito.whenNew; import java.io.ByteArrayInputStream; @@ -359,6 +359,9 @@ public class ClientConfigResourceProviderTest { expect(configHelper.createUserGroupsMap(stackId, cluster, desiredConfigMap)).andReturn(userGroupsMap).anyTimes(); PowerMock.expectNew(File.class, new Class<?>[]{String.class}, anyObject(String.class)).andReturn(newFile).anyTimes(); + PowerMock.mockStatic(File.class); + expect(File.createTempFile(anyString(), anyString(), anyObject(File.class))).andReturn(newFile); + String commandLine = "ambari-python-wrap /tmp/stacks/S1/V1/PIG/package/null generate_configs "+newFile + " /tmp/stacks/S1/V1/PIG/package /var/lib/ambari-server/tmp/structured-out.json " + "INFO /var/lib/ambari-server/tmp"; @@ -397,8 +400,7 @@ public class ClientConfigResourceProviderTest { Set<Resource> resources = provider.getResources(request, predicate); assertFalse(resources.isEmpty()); - String str = FileUtils.readFileToString(newFile); - assertTrue(str.contains("\"user_groups\":\"{\\\"hdfsUser\\\":[\\\"hdfsGroup\\\"]}")); + assertFalse(newFile.exists()); // verify verify(managementController, clusters, cluster, ambariMetaInfo, stackId, componentInfo,commandScriptDefinition, @@ -574,8 +576,9 @@ public class ClientConfigResourceProviderTest { userSet.add("hdfs"); expect(configHelper.getPropertyValuesWithPropertyType(stackId, PropertyInfo.PropertyType.USER, cluster, desiredConfigMap)).andReturn(userSet); PowerMock.expectNew(File.class, new Class<?>[]{String.class}, anyObject(String.class)).andReturn(mockFile).anyTimes(); + PowerMock.mockStatic(File.class); + expect(File.createTempFile(anyString(), anyString(), anyObject(File.class))).andReturn(PowerMock.createNiceMock(File.class)); PowerMock.createNiceMockAndExpectNew(PrintWriter.class, anyObject()); - expect(mockFile.getParent()).andReturn(""); PowerMock.mockStatic(Runtime.class); expect(mockFile.exists()).andReturn(true); String commandLine = "ambari-python-wrap " + commonServicesPath + "/PIG/package/null generate_configs null " + http://git-wip-us.apache.org/repos/asf/ambari/blob/fc36391c/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py index b2636ab..85098fa 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py @@ -32,6 +32,7 @@ from resource_management.libraries.script.script import Script @patch.object(tempfile,"mkdtemp", new = MagicMock(return_value='/tmp/123')) @patch.object(contextlib,"closing", new = MagicMock()) @patch("os.path.exists", new = MagicMock(return_value=True)) +@patch("os.chmod", new = MagicMock(return_value=True)) class Test(RMFTestCase): COMMON_SERVICES_PACKAGE_DIR = "HDFS/2.1.0.2.0/package" STACK_VERSION = "2.0.6" @@ -49,25 +50,25 @@ class Test(RMFTestCase): ) self.assertResourceCalled('XmlConfig', 'hdfs-site.xml', conf_dir = '/tmp/123', - mode=0644, + mode=0600, configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site'], configurations = self.getConfig()['configurations']['hdfs-site'], ) self.assertResourceCalled('File', '/tmp/123/hadoop-env.sh', - mode=0644, + mode=0600, content = InlineTemplate(self.getConfig()['configurations']['hadoop-env']['content']), ) self.assertResourceCalled('File', '/tmp/123/log4j.properties', - mode=0644, + mode=0600, content = InlineTemplate(self.getConfig()['configurations']['hdfs-log4j']['content']+ self.getConfig()['configurations']['yarn-log4j']['content']), ) self.assertResourceCalled('PropertiesFile', '/tmp/123/runtime.properties', - mode=0644, + mode=0600, properties = UnknownConfigurationMock(), ) self.assertResourceCalled('PropertiesFile', '/tmp/123/startup.properties', - mode=0644, + mode=0600, properties = UnknownConfigurationMock(), ) self.assertResourceCalled('Directory', '/tmp/123',