This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push: new d68616b YARN-10130. FS-CS converter: Do not allow output dir to be the same as input dir. Contributed by Adam Antal d68616b is described below commit d68616b0453881bec8dcb917a0b7d3fbe0838965 Author: Szilard Nemeth <snem...@apache.org> AuthorDate: Tue Feb 25 21:30:04 2020 +0100 YARN-10130. FS-CS converter: Do not allow output dir to be the same as input dir. Contributed by Adam Antal --- .../FSConfigToCSConfigArgumentHandler.java | 48 +++++++++++++++++ .../converter/FSConfigConverterTestCommons.java | 2 +- .../TestFSConfigToCSConfigArgumentHandler.java | 63 ++++++++++++++++++++++ 3 files changed, 112 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java index c751953..9121a6d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java @@ -25,6 +25,7 @@ import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,6 +40,13 @@ public class FSConfigToCSConfigArgumentHandler { private static final Logger LOG = LoggerFactory.getLogger(FSConfigToCSConfigArgumentHandler.class); + private static final String ALREADY_CONTAINS_EXCEPTION_MSG = + "The %s (provided with %s|%s arguments) contains " + + "the %s provided with the %s|%s options."; + private static final String ALREADY_CONTAINS_FILE_EXCEPTION_MSG = + "The %s %s (provided with %s|%s arguments) already contains a file " + + "or directory named %s which will be the output of the conversion!"; + private FSConfigToCSConfigRuleHandler ruleHandler; private FSConfigToCSConfigConverterParams converterParams; private ConversionOptions conversionOptions; @@ -213,6 +221,7 @@ public class FSConfigToCSConfigArgumentHandler { checkFile(CliOption.FAIR_SCHEDULER, fairSchedulerXmlFile); checkFile(CliOption.CONVERSION_RULES, conversionRulesFile); checkDirectory(CliOption.OUTPUT_DIR, outputDir); + checkOutputDirDoesNotContainXmls(yarnSiteXmlFile, outputDir); return FSConfigToCSConfigConverterParams.Builder.create() .withYarnSiteXmlConfig(yarnSiteXmlFile) @@ -225,6 +234,45 @@ public class FSConfigToCSConfigArgumentHandler { .build(); } + private static void checkOutputDirDoesNotContainXmls(String yarnSiteXmlFile, + String outputDir) { + if (yarnSiteXmlFile == null || outputDir == null) { + return; + } + + // check whether yarn-site.xml is not in the output folder + File xmlFile = new File(yarnSiteXmlFile); + File xmlParentFolder = xmlFile.getParentFile(); + File output = new File(outputDir); + if (output.equals(xmlParentFolder)) { + throw new IllegalArgumentException( + String.format(ALREADY_CONTAINS_EXCEPTION_MSG, + CliOption.OUTPUT_DIR.name, CliOption.OUTPUT_DIR.shortSwitch, + CliOption.OUTPUT_DIR.longSwitch, CliOption.YARN_SITE.name, + CliOption.YARN_SITE.shortSwitch, + CliOption.YARN_SITE.longSwitch)); + } + + // check whether the output folder does not contain nor yarn-site.xml + // neither capacity-scheduler.xml + checkFileNotInOutputDir(output, + YarnConfiguration.YARN_SITE_CONFIGURATION_FILE); + checkFileNotInOutputDir(output, + YarnConfiguration.CS_CONFIGURATION_FILE); + } + + private static void checkFileNotInOutputDir(File output, String fileName) { + File file = new File(output, fileName); + if (file.exists()) { + throw new IllegalArgumentException( + String.format(ALREADY_CONTAINS_FILE_EXCEPTION_MSG, + CliOption.OUTPUT_DIR.name, output, + CliOption.OUTPUT_DIR.shortSwitch, + CliOption.OUTPUT_DIR.longSwitch, + fileName)); + } + } + private void printHelp(Options opts) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("General options are: ", opts); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java index 4f67809..af915d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java @@ -35,7 +35,7 @@ import static org.junit.Assert.assertTrue; * */ public class FSConfigConverterTestCommons { - private final static String TEST_DIR = + public final static String TEST_DIR = new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath(); public final static String FS_ALLOC_FILE = new File(TEST_DIR, "test-fair-scheduler.xml").getAbsolutePath(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java index edf4c86..7a29c71 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java @@ -20,10 +20,13 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.io.File; import java.io.IOException; import java.util.Arrays; import java.util.List; +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -492,4 +495,64 @@ public class TestFSConfigToCSConfigArgumentHandler { assertFalse("No terminal rule check was enabled", conversionOptions.isNoRuleTerminalCheck()); } + + @Test + public void testYarnSiteOptionInOutputFolder() throws Exception { + setupFSConfigConversionFiles(true); + + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + String[] args = new String[] { + "-y", FSConfigConverterTestCommons.YARN_SITE_XML, + "-o", FSConfigConverterTestCommons.TEST_DIR}; + + int retVal = argumentHandler.parseAndConvert(args); + assertEquals("Return value", -1, retVal); + + assertTrue(fsTestCommons.getErrContent() + .toString().contains("contains the yarn-site.xml")); + } + + private void testFileExistsInOutputFolder(String file) throws Exception { + File testFile = new File(FSConfigConverterTestCommons.OUTPUT_DIR, file); + try { + FileUtils.touch(testFile); + + setupFSConfigConversionFiles(true); + + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + String[] args = new String[] { + "-y", FSConfigConverterTestCommons.YARN_SITE_XML, + "-o", FSConfigConverterTestCommons.OUTPUT_DIR}; + + int retVal = argumentHandler.parseAndConvert(args); + assertEquals("Return value", -1, retVal); + + String expectedMessage = String.format( + "already contains a file or directory named %s", file); + + assertTrue(fsTestCommons.getErrContent() + .toString().contains(expectedMessage)); + } finally { + if (testFile.exists()) { + testFile.delete(); + } + } + } + + @Test + public void testYarnSiteExistsInOutputFolder() throws Exception { + testFileExistsInOutputFolder( + YarnConfiguration.YARN_SITE_CONFIGURATION_FILE); + } + + @Test + public void testCapacitySchedulerXmlExistsInOutputFolder() + throws Exception { + testFileExistsInOutputFolder( + YarnConfiguration.CS_CONFIGURATION_FILE); + } } \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org