http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java deleted file mode 100644 index 057f6c5..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.common.tools; - -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; -import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationReportPBImpl; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** Test slider util methods. */ -public class TestSliderUtils { - protected static final Logger log = - LoggerFactory.getLogger(TestSliderUtils.class); - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - //@Test - public void testTruncate() { - Assert.assertEquals(SliderUtils.truncate(null, 5), null); - Assert.assertEquals(SliderUtils.truncate("323", -1), "323"); - Assert.assertEquals(SliderUtils.truncate("3232", 5), "3232"); - Assert.assertEquals(SliderUtils.truncate("1234567890", 0), "1234567890"); - Assert.assertEquals(SliderUtils.truncate("123456789012345", 15), "123456789012345"); - Assert.assertEquals(SliderUtils.truncate("123456789012345", 14), "12345678901..."); - Assert.assertEquals(SliderUtils.truncate("1234567890", 1), "1"); - Assert.assertEquals(SliderUtils.truncate("1234567890", 10), "1234567890"); - Assert.assertEquals(SliderUtils.truncate("", 10), ""); - } - - //@Test - public void testApplicationReportComparison() { - List<ApplicationReport> instances = getApplicationReports(); - - SliderUtils.sortApplicationsByMostRecent(instances); - - Assert.assertEquals(1000, instances.get(0).getStartTime()); - Assert.assertEquals(1000, instances.get(1).getStartTime()); - Assert.assertEquals(1000, instances.get(2).getStartTime()); - Assert.assertEquals(1000, instances.get(3).getStartTime()); - - instances = getApplicationReports(); - - SliderUtils.sortApplicationReport(instances); - Assert.assertEquals(1000, instances.get(0).getStartTime()); - Assert.assertEquals(1000, instances.get(1).getStartTime()); - Assert.assertEquals(1000, instances.get(2).getStartTime()); - Assert.assertEquals(1000, instances.get(3).getStartTime()); - - Assert.assertTrue(instances.get(0).getYarnApplicationState() == YarnApplicationState.ACCEPTED || - instances.get(0).getYarnApplicationState() == YarnApplicationState.RUNNING); - Assert.assertTrue(instances.get(1).getYarnApplicationState() == YarnApplicationState.ACCEPTED || - instances.get(1).getYarnApplicationState() == YarnApplicationState.RUNNING); - Assert.assertTrue(instances.get(2).getYarnApplicationState() == YarnApplicationState.ACCEPTED || - instances.get(2).getYarnApplicationState() == YarnApplicationState.RUNNING); - Assert.assertTrue(instances.get(3).getYarnApplicationState() == YarnApplicationState.KILLED); - } - - private List<ApplicationReport> getApplicationReports() { - List<ApplicationReport> instances = new ArrayList<ApplicationReport>(); - instances.add(getApplicationReport(1000, 0, "app1", YarnApplicationState.ACCEPTED)); - instances.add(getApplicationReport(900, 998, "app1", YarnApplicationState.KILLED)); - instances.add(getApplicationReport(900, 998, "app2", YarnApplicationState.FAILED)); - instances.add(getApplicationReport(1000, 0, "app2", YarnApplicationState.RUNNING)); - instances.add(getApplicationReport(800, 837, "app3", YarnApplicationState.FINISHED)); - instances.add(getApplicationReport(1000, 0, "app3", YarnApplicationState.RUNNING)); - instances.add(getApplicationReport(900, 998, "app3", YarnApplicationState.KILLED)); - instances.add(getApplicationReport(800, 837, "app4", YarnApplicationState.FINISHED)); - instances.add(getApplicationReport(1000, 1050, "app4", YarnApplicationState.KILLED)); - instances.add(getApplicationReport(900, 998, "app4", YarnApplicationState.FINISHED)); - - Assert.assertEquals("app1", instances.get(0).getApplicationType()); - Assert.assertEquals("app1", instances.get(1).getApplicationType()); - Assert.assertEquals("app2", instances.get(2).getApplicationType()); - Assert.assertEquals("app2", instances.get(3).getApplicationType()); - return instances; - } - - private ApplicationReportPBImpl getApplicationReport(long startTime, - long finishTime, - String name, - YarnApplicationState state) { - ApplicationReportPBImpl ar = new ApplicationReportPBImpl(); - ar.setFinishTime(finishTime); - ar.setStartTime(startTime); - ar.setApplicationType(name); - ar.setYarnApplicationState(state); - return ar; - } - - - //@Test - public void testGetHdpVersion() { - String hdpVersion = "2.3.2.0-2766"; - Assert.assertEquals("Version should be empty", null, - SliderUtils.getHdpVersion()); - } - - //@Test - public void testIsHdp() { - Assert.assertFalse("Should be false", SliderUtils.isHdp()); - } - - //@Test - public void testWrite() throws IOException { - File testWriteFile = folder.newFile("testWrite"); - SliderUtils.write(testWriteFile, "test".getBytes("UTF-8")); - Assert.assertTrue(FileUtils.readFileToString(testWriteFile, "UTF-8").equals("test")); - } -}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestWindowsSupport.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestWindowsSupport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestWindowsSupport.java deleted file mode 100644 index 073fd51..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestWindowsSupport.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.tools; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.ChecksumFileSystem; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.service.ServiceStateException; -import org.apache.hadoop.util.Shell; -import org.apache.slider.utils.YarnMiniClusterTestBase; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.FileNotFoundException; -import java.net.URI; -import java.util.Arrays; -import java.util.List; -import java.util.regex.Pattern; - -/** - * Test windows support. - */ -public class TestWindowsSupport extends YarnMiniClusterTestBase { - private static final Logger LOG = - LoggerFactory.getLogger(TestWindowsSupport.class); - - private static final Pattern HAS_DRIVE_LETTER_SPECIFIER = - Pattern.compile("^/?[a-zA-Z]:"); - public static final String WINDOWS_FILE = - "C:\\Users\\Administrator\\AppData\\Local\\Temp" + - "\\junit3180177850133852404\\testpkg\\appdef_1.zip"; - - - private static boolean hasWindowsDrive(String path) { - return HAS_DRIVE_LETTER_SPECIFIER.matcher(path).find(); - } - - private static int startPositionWithoutWindowsDrive(String path) { - if (hasWindowsDrive(path)) { - return path.charAt(0) == '/' ? 3 : 2; - } else { - return 0; - } - } - - //@Test - public void testHasWindowsDrive() throws Throwable { - assertTrue(hasWindowsDrive(WINDOWS_FILE)); - } - - //@Test - public void testStartPosition() throws Throwable { - assertEquals(2, startPositionWithoutWindowsDrive(WINDOWS_FILE)); - } - - //@Test - public void testPathHandling() throws Throwable { - assumeWindows(); - - Path path = new Path(WINDOWS_FILE); - URI uri = path.toUri(); - // assert "file" == uri.scheme - assertNull(uri.getAuthority()); - - Configuration conf = new Configuration(); - - FileSystem localfs = FileSystem.get(uri, conf); - assertTrue(localfs instanceof ChecksumFileSystem); - try { - FileStatus stat = localfs.getFileStatus(path); - fail("expected an exception, got " + stat); - } catch (FileNotFoundException fnfe) { - // expected - } - - try { - FSDataInputStream appStream = localfs.open(path); - } catch (FileNotFoundException fnfe) { - // expected - } - } - - //@Test - public void testExecNonexistentBinary() throws Throwable { - assumeWindows(); - List<String> commands = Arrays.asList("undefined-application", "--version"); - try { - exec(0, commands); - fail("expected an exception"); - } catch (ServiceStateException e) { - if (!(e.getCause() instanceof FileNotFoundException)) { - throw e; - } - } - } - //@Test - public void testExecNonexistentBinary2() throws Throwable { - assumeWindows(); - assertFalse(doesAppExist(Arrays.asList("undefined-application", - "--version"))); - } - - //@Test - public void testEmitKillCommand() throws Throwable { - - int result = killJavaProcesses("regionserver", 9); - // we know the exit code if there is no supported kill operation - assertTrue(getKillSupported() || result == -1); - } - - //@Test - public void testHadoopHomeDefined() throws Throwable { - assumeWindows(); - String hadoopHome = Shell.getHadoopHome(); - LOG.info("HADOOP_HOME={}", hadoopHome); - } - - //@Test - public void testHasWinutils() throws Throwable { - assumeWindows(); - SliderUtils.maybeVerifyWinUtilsValid(); - } - - //@Test - public void testExecWinutils() throws Throwable { - assumeWindows(); - String winUtilsPath = Shell.getWinUtilsPath(); - assertTrue(SliderUtils.isSet(winUtilsPath)); - File winUtils = new File(winUtilsPath); - LOG.debug("Winutils is at {}", winUtils); - - exec(0, Arrays.asList(winUtilsPath, "systeminfo")); - } - - //@Test - public void testPath() throws Throwable { - String path = extractPath(); - LOG.info("Path value = {}", path); - } - - //@Test - public void testFindJavac() throws Throwable { - String name = Shell.WINDOWS ? "javac.exe" : "javac"; - assertNotNull(locateExecutable(name)); - } - - //@Test - public void testHadoopDLL() throws Throwable { - assumeWindows(); - // split the path - File exepath = locateExecutable("HADOOP.DLL"); - assertNotNull(exepath); - LOG.info("Hadoop DLL at: {}", exepath); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestZKIntegration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestZKIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestZKIntegration.java deleted file mode 100644 index ed9337d..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestZKIntegration.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.tools; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.registry.server.services.MicroZookeeperServiceKeys; -import org.apache.slider.client.SliderClient; -import org.apache.slider.core.zk.ZKIntegration; -import org.apache.slider.utils.KeysForTests; -import org.apache.slider.utils.YarnZKMiniClusterTestBase; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooDefs; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.util.List; - -/** - * Test ZK integration. - */ -public class TestZKIntegration extends YarnZKMiniClusterTestBase implements - KeysForTests { - private static final Logger LOG = - LoggerFactory.getLogger(TestZKIntegration.class); - - public static final String USER = KeysForTests.USERNAME; - public static final int CONNECT_TIMEOUT = 5000; - private ZKIntegration zki; - - @Before - public void createCluster() { - Configuration conf = getConfiguration(); - String name = methodName.getMethodName(); - File zkdir = new File("target/zk/${name}"); - FileUtil.fullyDelete(zkdir); - conf.set(MicroZookeeperServiceKeys.KEY_ZKSERVICE_DIR, zkdir - .getAbsolutePath()); - createMicroZKCluster("-"+ name, conf); - } - - @After - public void closeZKI() throws IOException { - if (zki != null) { - zki.close(); - zki = null; - } - } - - public ZKIntegration initZKI() throws IOException, InterruptedException { - zki = createZKIntegrationInstance( - getZKBinding(), methodName.getMethodName(), true, false, - CONNECT_TIMEOUT); - return zki; - } - - //@Test - public void testListUserClustersWithoutAnyClusters() throws Throwable { - assertHasZKCluster(); - initZKI(); - String userPath = ZKIntegration.mkSliderUserPath(USER); - List<String> clusters = this.zki.getClusters(); - assertTrue(SliderUtils.isEmpty(clusters)); - } - - //@Test - public void testListUserClustersWithOneCluster() throws Throwable { - assertHasZKCluster(); - - initZKI(); - String userPath = ZKIntegration.mkSliderUserPath(USER); - String fullPath = zki.createPath(userPath, "/cluster-", - ZooDefs.Ids.OPEN_ACL_UNSAFE, - CreateMode.EPHEMERAL_SEQUENTIAL); - LOG.info("Ephemeral path {}", fullPath); - List<String> clusters = zki.getClusters(); - assertEquals(1, clusters.size()); - assertTrue(fullPath.endsWith(clusters.get(0))); - } - - //@Test - public void testListUserClustersWithTwoCluster() throws Throwable { - initZKI(); - String userPath = ZKIntegration.mkSliderUserPath(USER); - String c1 = createEphemeralChild(zki, userPath); - LOG.info("Ephemeral path $c1"); - String c2 = createEphemeralChild(zki, userPath); - LOG.info("Ephemeral path $c2"); - List<String> clusters = zki.getClusters(); - assertEquals(2, clusters.size()); - assertTrue((c1.endsWith(clusters.get(0)) && c2.endsWith(clusters.get(1))) || - (c1.endsWith(clusters.get(1)) && c2.endsWith(clusters.get(0)))); - } - - //@Test - public void testCreateAndDeleteDefaultZKPath() throws Throwable { - MockSliderClient client = new MockSliderClient(); - - String path = client.createZookeeperNodeInner("cl1", true); - zki = client.getLastZKIntegration(); - - String zkPath = ZKIntegration.mkClusterPath(USER, "cl1"); - assertEquals("zkPath must be as expected", zkPath, - "/services/slider/users/" + USER + "/cl1"); - assertEquals(path, zkPath); - assertNull("ZKIntegration should be null.", zki); - zki = createZKIntegrationInstance(getZKBinding(), "cl1", true, false, - CONNECT_TIMEOUT); - assertFalse(zki.exists(zkPath)); - - path = client.createZookeeperNodeInner("cl1", false); - zki = client.getLastZKIntegration(); - assertNotNull(zki); - assertEquals("zkPath must be as expected", zkPath, - "/services/slider/users/" + USER + "/cl1"); - assertEquals(path, zkPath); - assertTrue(zki.exists(zkPath)); - zki.createPath(zkPath, "/cn", ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode - .PERSISTENT); - assertTrue(zki.exists(zkPath + "/cn")); - client.deleteZookeeperNode("cl1"); - assertFalse(zki.exists(zkPath)); - } - - public static String createEphemeralChild(ZKIntegration zki, String userPath) - throws KeeperException, InterruptedException { - return zki.createPath(userPath, "/cluster-", - ZooDefs.Ids.OPEN_ACL_UNSAFE, - CreateMode.EPHEMERAL_SEQUENTIAL); - } - - /** - * Test slider client that overriddes ZK client. - */ - public class MockSliderClient extends SliderClient { - private ZKIntegration zki; - - @Override - public String getUsername() { - return USER; - } - - @Override - protected ZKIntegration getZkClient(String clusterName, String user) { - try { - zki = createZKIntegrationInstance(getZKBinding(), clusterName, true, - false, CONNECT_TIMEOUT); - } catch (Exception e) { - fail("creating ZKIntergration threw an exception"); - } - return zki; - } - - @Override - public Configuration getConfig() { - return new Configuration(); - } - - public ZKIntegration getLastZKIntegration() { - return zki; - } - - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java deleted file mode 100644 index 54c3576..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.core.registry.docstore; - -import com.google.common.base.Charsets; -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.fs.Path; -import org.apache.slider.common.tools.SliderFileSystem; -import org.codehaus.jackson.map.ObjectMapper; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.powermock.api.easymock.PowerMock; -import org.yaml.snakeyaml.Yaml; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -import static org.easymock.EasyMock.anyObject; -import static org.easymock.EasyMock.expect; -import static org.mockito.Matchers.anyString; -import static org.powermock.api.easymock.PowerMock.createNiceMock; - -public class TestPublishedConfigurationOutputter { - private static HashMap<String, String> config = new HashMap<>(); - - @Rule - public TemporaryFolder tmpDir = new TemporaryFolder(); - - @Before - public void setup() { - config.put("key1", "val1"); - } - - //@Test - public void testJson() throws IOException { - PublishedConfigurationOutputter configurationOutputter = - PublishedConfigurationOutputter.createOutputter(ConfigFormat.JSON, - new PublishedConfiguration("description", - config.entrySet())); - - String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)", - ""); - assert "{\"key1\":\"val1\"}".equals(output); - - File file = tmpDir.newFile(); - configurationOutputter.save(file); - - ObjectMapper mapper = new ObjectMapper(); - @SuppressWarnings("unchecked") - Map<String, String> read = mapper.readValue(file, Map.class); - assert 1 == read.size(); - assert "val1".equals(read.get("key1")); - } - - //@Test - public void testXml() throws IOException { - PublishedConfigurationOutputter configurationOutputter = - PublishedConfigurationOutputter.createOutputter(ConfigFormat.XML, - new PublishedConfiguration("description", - config.entrySet())); - - String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)", - ""); - assert output.contains("<name>key1</name><value>val1</value>"); - - File file = tmpDir.newFile(); - configurationOutputter.save(file); - - assert FileUtils.readFileToString(file, Charsets.UTF_8) - .replaceAll("( |\\r|\\n)", "") - .contains("<name>key1</name><value>val1</value>"); - } - - //@Test - public void testHadoopXml() throws IOException { - PublishedConfigurationOutputter configurationOutputter = - PublishedConfigurationOutputter.createOutputter(ConfigFormat.HADOOP_XML, - new PublishedConfiguration("description", - config.entrySet())); - - String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)", - ""); - assert output.contains("<name>key1</name><value>val1</value>"); - - File file = tmpDir.newFile(); - configurationOutputter.save(file); - - assert FileUtils.readFileToString(file, Charsets.UTF_8) - .replaceAll("( |\\r|\\n)", "") - .contains("<name>key1</name><value>val1</value>"); - } - - //@Test - public void testProperties() throws IOException { - PublishedConfigurationOutputter configurationOutputter = - PublishedConfigurationOutputter.createOutputter(ConfigFormat.PROPERTIES, - new PublishedConfiguration("description", - config.entrySet())); - - String output = configurationOutputter.asString(); - assert output.contains("key1=val1"); - - File file = tmpDir.newFile(); - configurationOutputter.save(file); - - Properties properties = new Properties(); - FileInputStream fis = null; - try { - fis = new FileInputStream(file); - properties.load(fis); - } finally { - if (fis != null) { - fis.close(); - } - } - assert 1 == properties.size(); - assert "val1".equals(properties.getProperty("key1")); - } - - //@Test - public void testYaml() throws IOException { - PublishedConfigurationOutputter configurationOutputter = - PublishedConfigurationOutputter.createOutputter(ConfigFormat.YAML, - new PublishedConfiguration("description", - config.entrySet())); - - String output = configurationOutputter.asString().replaceAll("(\\r|\\n)", - ""); - assert "key1: val1".equals(output); - - File file = tmpDir.newFile(); - configurationOutputter.save(file); - - Yaml yaml = new Yaml(); - FileInputStream fis = null; - Map<String, String> read; - try { - fis = new FileInputStream(file); - read = (Map<String, String>) yaml.load(fis); - } finally { - if (fis != null) { - fis.close(); - } - } - assert 1 == read.size(); - assert "val1".equals(read.get("key1")); - } - - //@Test - public void testEnv() throws IOException { - HashMap<String, String> envConfig = new HashMap<>(config); - envConfig.put("content", "content {{key1}} "); - - PublishedConfigurationOutputter configurationOutputter = - PublishedConfigurationOutputter.createOutputter(ConfigFormat.ENV, - new PublishedConfiguration("description", - envConfig.entrySet())); - - String output = configurationOutputter.asString(); - assert "content val1 ".equals(output); - - File file = tmpDir.newFile(); - configurationOutputter.save(file); - - assert "content val1 ".equals(FileUtils.readFileToString(file, - Charsets.UTF_8)); - } - - //@Test - public void testTemplate1() throws IOException { - HashMap<String, String> templateConfig = new HashMap<>(config); - templateConfig.put(ConfigUtils.TEMPLATE_FILE, "templateFileName"); - - SliderFileSystem fileSystem = createNiceMock(SliderFileSystem.class); - expect(fileSystem.buildResourcePath(anyString())).andReturn(new Path("path")).anyTimes(); - expect(fileSystem.isFile(anyObject(Path.class))).andReturn(true).anyTimes(); - expect(fileSystem.cat(anyObject(Path.class))).andReturn("content {{key1}}\n more ${key1} content").anyTimes(); - - PowerMock.replay(fileSystem); - - ConfigUtils.prepConfigForTemplateOutputter(ConfigFormat.TEMPLATE, - templateConfig, fileSystem, "clusterName", null); - PublishedConfigurationOutputter configurationOutputter = - PublishedConfigurationOutputter.createOutputter(ConfigFormat.TEMPLATE, - new PublishedConfiguration("description", - templateConfig.entrySet())); - - String output = configurationOutputter.asString(); - assert "content val1\n more val1 content".equals(output); - - File file = tmpDir.newFile(); - configurationOutputter.save(file); - - PowerMock.verify(fileSystem); - - assert "content val1\n more val1 content".equals( - FileUtils.readFileToString(file, Charsets.UTF_8)); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestFilesystemPermissions.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestFilesystemPermissions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestFilesystemPermissions.java deleted file mode 100644 index 1e5d769..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestFilesystemPermissions.java +++ /dev/null @@ -1,263 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.other; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileContext; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.UnsupportedFileSystemException; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.util.DiskChecker; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService; -import org.apache.slider.utils.YarnMiniClusterTestBase; -import org.junit.After; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * This test class exists to look at permissions of the filesystem, especially - * that created by Mini YARN clusters. On some windows jenkins machines, - * YARN actions were failing as the directories had the wrong permissions - * (i.e. too lax) - */ -public class TestFilesystemPermissions extends YarnMiniClusterTestBase { - - private static final Logger LOG = LoggerFactory.getLogger( - TestFilesystemPermissions.class); - - private List<File> filesToDelete = new ArrayList<>(); - - @After - public void deleteFiles() { - for (File f : filesToDelete) { - FileUtil.fullyDelete(f, true); - } - } - - //@Test - public void testJavaFSOperations() throws Throwable { - assertNativeLibrariesPresent(); - File subdir = testDir(); - subdir.mkdir(); - assertTrue(subdir.isDirectory()); - assertTrue(FileUtil.canRead(subdir)); - assertTrue(FileUtil.canWrite(subdir)); - assertTrue(FileUtil.canExecute(subdir)); - } - - //@Test - public void testDiskCheckerOperations() throws Throwable { - assertNativeLibrariesPresent(); - File subdir = testDir(); - subdir.mkdir(); - DiskChecker checker = new DiskChecker(); - checker.checkDir(subdir); - } - - //@Test - public void testDiskCheckerMkdir() throws Throwable { - assertNativeLibrariesPresent(); - File subdir = testDir(); - subdir.mkdirs(); - DiskChecker checker = new DiskChecker(); - checker.checkDir(subdir); - } - - /** - * Get a test dir for this method; one that will be deleted on teardown. - * @return a filename unique to this test method - */ - File testDir() { - File parent = new File("target/testfspermissions"); - parent.mkdir(); - File testdir = new File(parent, methodName.getMethodName()); - filesToDelete.add(testdir); - return testdir; - } - - - //@Test - public void testPermsMap() throws Throwable { - File dir = testDir(); - String diruri = dir.toURI().toString(); - FileContext lfs = createLocalFS(dir, getConfiguration()); - getLocalDirsPathPermissionsMap(lfs, diruri); - } - - //@Test - public void testInitLocaldir() throws Throwable { - File dir = testDir(); - String diruri = dir.toURI().toString(); - FileContext lfs = createLocalFS(dir, getConfiguration()); - initializeLocalDir(lfs, diruri); - List<String> localDirs = getInitializedLocalDirs(lfs, Arrays.asList( - diruri)); - assertEquals(1, localDirs.size()); - } - - - //@Test - public void testValidateMiniclusterPerms() throws Throwable { - int numLocal = 1; - String cluster = createMiniCluster("", getConfiguration(), 1, numLocal, 1, - false); - File workDir = getMiniCluster().getTestWorkDir(); - List<File> localdirs = new ArrayList<>(); - for (File file : workDir.listFiles()) { - if (file.isDirectory() && file.getAbsolutePath().contains("-local")) { - // local dir - localdirs.add(file); - } - } - assertEquals(numLocal, localdirs.size()); - FileContext lfs = createLocalFS(workDir, getConfiguration()); - for (File file : localdirs) { - checkLocalDir(lfs, file.toURI().toString()); - } - } - - FileContext createLocalFS(File dir, Configuration conf) - throws UnsupportedFileSystemException { - return FileContext.getFileContext(dir.toURI(), conf); - } - - /** - * Extracted from ResourceLocalizationService. - * @param lfs - * @param localDir - * @return perms map - * @see ResourceLocalizationService - */ - private Map<Path, FsPermission> getLocalDirsPathPermissionsMap( - FileContext lfs, - String localDir) { - Map<Path, FsPermission> localDirPathFsPermissionsMap = new HashMap<>(); - - FsPermission defaultPermission = - FsPermission.getDirDefault().applyUMask(lfs.getUMask()); - FsPermission nmPrivatePermission = - ResourceLocalizationService.NM_PRIVATE_PERM.applyUMask(lfs.getUMask()); - - Path userDir = new Path(localDir, ContainerLocalizer.USERCACHE); - Path fileDir = new Path(localDir, ContainerLocalizer.FILECACHE); - Path sysDir = new Path( - localDir, - ResourceLocalizationService.NM_PRIVATE_DIR); - - localDirPathFsPermissionsMap.put(userDir, defaultPermission); - localDirPathFsPermissionsMap.put(fileDir, defaultPermission); - localDirPathFsPermissionsMap.put(sysDir, nmPrivatePermission); - return localDirPathFsPermissionsMap; - } - - private boolean checkLocalDir(FileContext lfs, String localDir) - throws IOException { - - Map<Path, FsPermission> pathPermissionMap = - getLocalDirsPathPermissionsMap(lfs, localDir); - - for (Map.Entry<Path, FsPermission> entry : pathPermissionMap.entrySet()) { - FileStatus status; - status = lfs.getFileStatus(entry.getKey()); - - if (!status.getPermission().equals(entry.getValue())) { - String msg = - "Permissions incorrectly set for dir " + entry.getKey() + - ", should be " + entry.getValue() + ", actual value = " + - status.getPermission(); - throw new YarnRuntimeException(msg); - } - } - return true; - } - - - private void initializeLocalDir(FileContext lfs, String localDir) - throws IOException { - - Map<Path, FsPermission> pathPermissionMap = - getLocalDirsPathPermissionsMap(lfs, localDir); - for (Map.Entry<Path, FsPermission> entry : pathPermissionMap.entrySet()) { - FileStatus status; - try { - status = lfs.getFileStatus(entry.getKey()); - } catch (FileNotFoundException fs) { - status = null; - } - - if (status == null) { - lfs.mkdir(entry.getKey(), entry.getValue(), true); - status = lfs.getFileStatus(entry.getKey()); - } - FsPermission perms = status.getPermission(); - if (!perms.equals(entry.getValue())) { - lfs.setPermission(entry.getKey(), entry.getValue()); - } - } - } - - synchronized private List<String> getInitializedLocalDirs(FileContext lfs, - List<String> dirs) throws IOException { - List<String> checkFailedDirs = new ArrayList<String>(); - for (String dir : dirs) { - try { - checkLocalDir(lfs, dir); - } catch (YarnRuntimeException e) { - checkFailedDirs.add(dir); - } - } - for (String dir : checkFailedDirs) { - LOG.info("Attempting to initialize " + dir); - initializeLocalDir(lfs, dir); - checkLocalDir(lfs, dir); - } - return dirs; - } - - - private void createDir(FileContext localFs, Path dir, FsPermission perm) - throws IOException { - if (dir == null) { - return; - } - try { - localFs.getFileStatus(dir); - } catch (FileNotFoundException e) { - createDir(localFs, dir.getParent(), perm); - localFs.mkdir(dir, perm, false); - if (!perm.equals(perm.applyUMask(localFs.getUMask()))) { - localFs.setPermission(dir, perm); - } - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestLocalDirStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestLocalDirStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestLocalDirStatus.java deleted file mode 100644 index a45b27d..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestLocalDirStatus.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.other; - -import org.apache.slider.utils.SliderTestUtils; -import org.apache.slider.utils.TestUtility; -import org.junit.Test; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; - -/** - * This test exists to diagnose local FS permissions. - */ -public class TestLocalDirStatus extends SliderTestUtils { - - - public static final int SIZE = 0x200000; - - //@Test - public void testTempDir() throws Throwable { - File tmpf = null; - try { - tmpf = File.createTempFile("testl", ".bin"); - createAndReadFile(tmpf, SIZE); - tmpf.delete(); - assertFalse(tmpf.exists()); - } finally { - if (tmpf != null) { - tmpf.delete(); - } - } - } - - //@Test - public void testTargetDir() throws Throwable { - File target = target(); - File tmpf = null; - try { - tmpf = File.createTempFile("testl", ".bin", target); - createAndReadFile(tmpf, SIZE); - tmpf.delete(); - assertFalse(tmpf.exists()); - } finally { - if (tmpf != null) { - tmpf.delete(); - } - - } - } - - public File target() { - File target = new File("target").getAbsoluteFile(); - assertTrue(target.exists()); - return target; - } - - //@Test - public void testRenameInTargetDir() throws Throwable { - File target = target(); - File tmpf = null; - File dst= null; - try { - tmpf = File.createTempFile("testl", ".bin", target); - dst = File.createTempFile("test-dest", ".bin", target); - createRenameAndReadFile(tmpf, dst, SIZE); - assertFalse(tmpf.exists()); - dst.delete(); - } finally { - if (tmpf != null) { - tmpf.delete(); - } - if (dst != null) { - dst.delete(); - } - } - } - - //@Test - public void testRenameInTmpDir() throws Throwable { - File tmpf = null; - File dst= null; - try { - tmpf = File.createTempFile("testl", ".bin"); - dst = File.createTempFile("test-dest", ".bin"); - createRenameAndReadFile(tmpf, dst, SIZE); - assertFalse(tmpf.exists()); - dst.delete(); - } finally { - if (tmpf != null) { - tmpf.delete(); - } - if (dst != null) { - dst.delete(); - } - } - } - - protected void createAndReadFile(File path, int len) throws IOException { - byte[] dataset = TestUtility.dataset(len, 32, 128); - writeFile(path, dataset); - assertTrue(path.exists()); - assertEquals(len, path.length()); - byte[] persisted = readFile(path); - TestUtility.compareByteArrays(dataset, persisted, len); - } - - protected void createRenameAndReadFile(File src, File dst, int len) - throws IOException { - byte[] dataset = TestUtility.dataset(len, 32, 128); - writeFile(src, dataset); - assertTrue(src.exists()); - assertEquals(len, src.length()); - dst.delete(); - assertFalse(dst.exists()); - assertTrue(src.renameTo(dst)); - assertEquals(len, dst.length()); - byte[] persisted = readFile(dst); - TestUtility.compareByteArrays(dataset, persisted, len); - } - - protected void writeFile(File path, byte[] dataset) - throws IOException { - FileOutputStream out = new FileOutputStream(path); - try { - out.write(dataset); - out.flush(); - } finally { - out.close(); - } - } - - protected byte[] readFile(File path) throws IOException { - assertTrue(path.getAbsoluteFile().exists()); - assertTrue(path.getAbsoluteFile().isFile()); - int len = (int)path.length(); - byte[] dataset = new byte[len]; - FileInputStream ins = new FileInputStream(path); - try { - ins.read(dataset); - } finally { - ins.close(); - } - return dataset; - } - - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestAbstractClientProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestAbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestAbstractClientProvider.java deleted file mode 100644 index 7bb8707..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestAbstractClientProvider.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.yarn.service.provider.AbstractClientProvider; -import org.apache.slider.api.resource.Artifact; -import org.apache.slider.api.resource.ConfigFile; -import org.apache.slider.api.resource.ConfigFile.TypeEnum; -import org.junit.Assert; -import org.junit.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import static org.easymock.EasyMock.anyObject; -import static org.easymock.EasyMock.createNiceMock; -import static org.easymock.EasyMock.expect; -import static org.easymock.EasyMock.replay; - -/** - * Test the AbstractClientProvider shared methods. - */ -public class TestAbstractClientProvider { - private static final String EXCEPTION_PREFIX = "Should have thrown " + - "exception: "; - private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " + - "exception: "; - - private static class ClientProvider extends AbstractClientProvider { - @Override - public void validateArtifact(Artifact artifact, FileSystem fileSystem) - throws IOException { - } - - @Override - protected void validateConfigFile(ConfigFile configFile, - FileSystem fileSystem) throws IOException { - } - } - - //@Test - public void testConfigFiles() throws IOException { - ClientProvider clientProvider = new ClientProvider(); - FileSystem mockFs = createNiceMock(FileSystem.class); - expect(mockFs.exists(anyObject(Path.class))).andReturn(true).anyTimes(); - replay(mockFs); - - ConfigFile configFile = new ConfigFile(); - List<ConfigFile> configFiles = new ArrayList<>(); - configFiles.add(configFile); - - try { - clientProvider.validateConfigFiles(configFiles, mockFs); - Assert.fail(EXCEPTION_PREFIX + "null file type"); - } catch (IllegalArgumentException e) { - } - - configFile.setType(TypeEnum.TEMPLATE); - try { - clientProvider.validateConfigFiles(configFiles, mockFs); - Assert.fail(EXCEPTION_PREFIX + "empty src_file for type template"); - } catch (IllegalArgumentException e) { - } - - configFile.setSrcFile("srcfile"); - try { - clientProvider.validateConfigFiles(configFiles, mockFs); - Assert.fail(EXCEPTION_PREFIX + "empty dest file"); - } catch (IllegalArgumentException e) { - } - - configFile.setDestFile("destfile"); - try { - clientProvider.validateConfigFiles(configFiles, mockFs); - } catch (IllegalArgumentException e) { - Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); - } - - configFile = new ConfigFile(); - configFile.setType(TypeEnum.JSON); - configFile.setSrcFile(null); - configFile.setDestFile("path/destfile2"); - configFiles.add(configFile); - try { - clientProvider.validateConfigFiles(configFiles, mockFs); - Assert.fail(EXCEPTION_PREFIX + "dest file with multiple path elements"); - } catch (IllegalArgumentException e) { - } - - configFile.setDestFile("/path/destfile2"); - try { - clientProvider.validateConfigFiles(configFiles, mockFs); - } catch (IllegalArgumentException e) { - Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); - } - - configFile.setDestFile("destfile"); - try { - clientProvider.validateConfigFiles(configFiles, mockFs); - Assert.fail(EXCEPTION_PREFIX + "duplicate dest file"); - } catch (IllegalArgumentException e) { - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestProviderFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestProviderFactory.java deleted file mode 100644 index c51eee2..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestProviderFactory.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers; - -import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultClientProvider; -import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderFactory; -import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderService; -import org.apache.hadoop.yarn.service.provider.ProviderFactory; -import org.apache.slider.api.resource.Artifact; -import org.apache.slider.api.resource.Artifact.TypeEnum; -import org.apache.hadoop.yarn.service.provider.docker.DockerClientProvider; -import org.apache.hadoop.yarn.service.provider.docker.DockerProviderFactory; -import org.apache.hadoop.yarn.service.provider.docker.DockerProviderService; -import org.apache.hadoop.yarn.service.provider.tarball.TarballClientProvider; -import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderFactory; -import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderService; -import org.junit.Test; - -import static org.junit.Assert.assertTrue; - -/** - * Test provider factories. - */ -public class TestProviderFactory { - //@Test - public void testDockerFactory() throws Throwable { - ProviderFactory factory = ProviderFactory - .createSliderProviderFactory(new Artifact().type(TypeEnum.DOCKER)); - assertTrue(factory instanceof DockerProviderFactory); - assertTrue(factory.createClientProvider() instanceof DockerClientProvider); - assertTrue(factory.createServerProvider() instanceof DockerProviderService); - assertTrue(ProviderFactory.getProviderService(new Artifact() - .type(TypeEnum.DOCKER)) instanceof DockerProviderService); - } - - //@Test - public void testTarballFactory() throws Throwable { - ProviderFactory factory = ProviderFactory - .createSliderProviderFactory(new Artifact().type(TypeEnum.TARBALL)); - assertTrue(factory instanceof TarballProviderFactory); - assertTrue(factory.createClientProvider() instanceof TarballClientProvider); - assertTrue(factory.createServerProvider() instanceof - TarballProviderService); - assertTrue(ProviderFactory.getProviderService(new Artifact() - .type(TypeEnum.TARBALL)) instanceof TarballProviderService); - } - - //@Test - public void testDefaultFactory() throws Throwable { - ProviderFactory factory = ProviderFactory - .createSliderProviderFactory(null); - assertTrue(factory instanceof DefaultProviderFactory); - assertTrue(factory.createClientProvider() instanceof DefaultClientProvider); - assertTrue(factory.createServerProvider() instanceof DefaultProviderService); - assertTrue(ProviderFactory.getProviderService(null) instanceof - DefaultProviderService); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestConfigSetNaming.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestConfigSetNaming.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestConfigSetNaming.java deleted file mode 100644 index 5743119..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestConfigSetNaming.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.registry; - -import org.apache.slider.core.registry.docstore.PublishedConfigSet; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Arrays; - -/** - * Test config set name validation. - */ -public class TestConfigSetNaming { - - void assertValid(String name) { - PublishedConfigSet.validateName(name); - } - - void assertInvalid(String name) { - try { - PublishedConfigSet.validateName(name); - Assert.fail("Invalid name was unexpectedly parsed: " + name); - } catch (IllegalArgumentException expected) { - // expected - } - } - - //@Test - public void testLowerCase() throws Throwable { - assertValid("abcdefghijklmnopqrstuvwxyz"); - } - - //@Test - public void testUpperCaseInvalid() throws Throwable { - assertInvalid("ABCDEFGHIJKLMNOPQRSTUVWXYZ"); - } - - //@Test - public void testNumbers() throws Throwable { - assertValid("01234567890"); - } - - //@Test - public void testChars() throws Throwable { - assertValid("a-_+"); - } - - //@Test - public void testInvalids() throws Throwable { - for (String s : Arrays.asList( - "", - " ", - "*", - "a/b", - "b\\a", - "\"", - "'", - "\u0000", - "\u0f00", - "key.value", - "-", - "+", - "_", - "?")) { - assertInvalid(s); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestRegistryPaths.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestRegistryPaths.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestRegistryPaths.java deleted file mode 100644 index 5e6b650..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestRegistryPaths.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.registry; - -import org.apache.hadoop.registry.client.binding.RegistryUtils; -import org.apache.slider.core.registry.SliderRegistryUtils; -import org.apache.slider.utils.SliderTestUtils; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Test registry paths. - */ -public class TestRegistryPaths { - - //@Test - public void testHomedirKerberos() throws Throwable { - String home = RegistryUtils.homePathForUser("hb...@hadoop.apache.org"); - try { - assertEquals("/users/hbase", home); - } catch (AssertionError e) { - SliderTestUtils.skip("homedir filtering not yet in hadoop registry " + - "module"); - } - } - - //@Test - public void testHomedirKerberosHost() throws Throwable { - String home = RegistryUtils.homePathForUser("hbase/localhost@HADOOP" + - ".APACHE.ORG"); - try { - assertEquals("/users/hbase", home); - } catch (AssertionError e) { - SliderTestUtils.skip("homedir filtering not yet in hadoop registry " + - "module"); - } - } - - //@Test - public void testRegistryPathForInstance() throws Throwable { - String path = SliderRegistryUtils.registryPathForInstance("instance"); - assertTrue(path.endsWith("/instance")); - } - - //@Test - public void testPathResolution() throws Throwable { - String home = RegistryUtils.homePathForCurrentUser(); - assertEquals(home, SliderRegistryUtils.resolvePath("~")); - assertEquals(home +"/", SliderRegistryUtils.resolvePath("~/")); - assertEquals(home +"/something", SliderRegistryUtils.resolvePath( - "~/something")); - assertEquals("~unresolved", SliderRegistryUtils.resolvePath( - "~unresolved")); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/actions/TestActions.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/actions/TestActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/actions/TestActions.java deleted file mode 100644 index a63f4b6..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/actions/TestActions.java +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.actions; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.service.ServiceOperations; -import org.apache.slider.server.appmaster.SliderAppMaster; -import org.apache.slider.server.appmaster.state.AppState; -import org.apache.slider.server.services.workflow.ServiceThreadFactory; -import org.apache.slider.server.services.workflow.WorkflowExecutorService; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * Test AM actions. - */ -public class TestActions { - protected static final Logger LOG = - LoggerFactory.getLogger(TestActions.class); - - private QueueService queues; - private WorkflowExecutorService<ExecutorService> executorService; - - - @Before - public void createService() { - queues = new QueueService(); - - Configuration conf = new Configuration(); - queues.init(conf); - - queues.start(); - - executorService = new WorkflowExecutorService<>("AmExecutor", - Executors.newCachedThreadPool( - new ServiceThreadFactory("AmExecutor", true))); - - executorService.init(conf); - executorService.start(); - } - - @After - public void destroyService() { - ServiceOperations.stop(executorService); - ServiceOperations.stop(queues); - } - - //@Test - public void testBasicService() throws Throwable { - queues.start(); - } - - //@Test - public void testDelayLogic() throws Throwable { - ActionNoteExecuted action = new ActionNoteExecuted("", 1000); - long now = System.currentTimeMillis(); - - long delay = action.getDelay(TimeUnit.MILLISECONDS); - assertTrue(delay >= 800); - assertTrue(delay <= 1800); - - ActionNoteExecuted a2 = new ActionNoteExecuted("a2", 10000); - assertTrue(action.compareTo(a2) < 0); - assertTrue(a2.compareTo(action) > 0); - assertEquals(0, action.compareTo(action)); - - } - - //@Test - public void testActionDelayedExecutorTermination() throws Throwable { - long start = System.currentTimeMillis(); - - ActionStopQueue stopAction = new ActionStopQueue(1000); - queues.scheduledActions.add(stopAction); - queues.run(); - AsyncAction take = queues.actionQueue.take(); - assertEquals(take, stopAction); - long stop = System.currentTimeMillis(); - assertTrue(stop - start > 500); - assertTrue(stop - start < 1500); - } - - //@Test - public void testImmediateQueue() throws Throwable { - ActionNoteExecuted noteExecuted = new ActionNoteExecuted("executed", 0); - queues.put(noteExecuted); - queues.put(new ActionStopQueue(0)); - QueueExecutor ex = new QueueExecutor(queues); - ex.run(); - assertTrue(queues.actionQueue.isEmpty()); - assertTrue(noteExecuted.executed.get()); - } - - //@Test - public void testActionOrdering() throws Throwable { - - ActionNoteExecuted note1 = new ActionNoteExecuted("note1", 500); - ActionStopQueue stop = new ActionStopQueue(1500); - ActionNoteExecuted note2 = new ActionNoteExecuted("note2", 800); - - List<AsyncAction> actions = Arrays.asList(note1, stop, note2); - Collections.sort(actions); - assertEquals(actions.get(0), note1); - assertEquals(actions.get(1), note2); - assertEquals(actions.get(2), stop); - } - - //@Test - public void testDelayedQueueWithReschedule() throws Throwable { - - ActionNoteExecuted note1 = new ActionNoteExecuted("note1", 500); - ActionStopQueue stop = new ActionStopQueue(1500); - ActionNoteExecuted note2 = new ActionNoteExecuted("note2", 800); - - assertTrue(note2.compareTo(stop) < 0); - assertTrue(note1.getNanos() < note2.getNanos()); - assertTrue(note2.getNanos() < stop.getNanos()); - queues.schedule(note1); - queues.schedule(note2); - queues.schedule(stop); - // async to sync expected to run in order - runQueuesToCompletion(); - assertTrue(note1.executed.get()); - assertTrue(note2.executed.get()); - } - - public void runQueuesToCompletion() { - queues.run(); - assertTrue(queues.scheduledActions.isEmpty()); - assertFalse(queues.actionQueue.isEmpty()); - QueueExecutor ex = new QueueExecutor(queues); - ex.run(); - // flush all stop commands from the queue - queues.flushActionQueue(ActionStopQueue.class); - - assertTrue(queues.actionQueue.isEmpty()); - } - - //@Test - public void testRenewedActionFiresOnceAtLeast() throws Throwable { - ActionNoteExecuted note1 = new ActionNoteExecuted("note1", 500); - RenewingAction renewer = new RenewingAction( - note1, - 500, - 100, - TimeUnit.MILLISECONDS, - 3); - queues.schedule(renewer); - ActionStopQueue stop = new ActionStopQueue(4, TimeUnit.SECONDS); - queues.schedule(stop); - // this runs all the delayed actions FIRST, so can't be used - // to play tricks of renewing actions ahead of the stop action - runQueuesToCompletion(); - assertEquals(1, renewer.executionCount.intValue()); - assertEquals(1, note1.executionCount.intValue()); - // assert the renewed item is back in - assertTrue(queues.scheduledActions.contains(renewer)); - } - - - //@Test - public void testRenewingActionOperations() throws Throwable { - ActionNoteExecuted note1 = new ActionNoteExecuted("note1", 500); - RenewingAction renewer = new RenewingAction( - note1, - 100, - 100, - TimeUnit.MILLISECONDS, - 3); - queues.renewing("note", renewer); - assertTrue(queues.removeRenewingAction("note")); - queues.stop(); - assertTrue(queues.waitForServiceToStop(10000)); - } - - /** - * Test action. - */ - public class ActionNoteExecuted extends AsyncAction { - private final AtomicBoolean executed = new AtomicBoolean(false); - private final AtomicLong executionTimeNanos = new AtomicLong(); - private final AtomicLong executionCount = new AtomicLong(); - - public ActionNoteExecuted(String text, int delay) { - super(text, delay); - } - - @Override - public void execute( - SliderAppMaster appMaster, - QueueAccess queueService, - AppState appState) throws Exception { - LOG.info("Executing {}", name); - executed.set(true); - executionTimeNanos.set(System.nanoTime()); - executionCount.incrementAndGet(); - LOG.info(this.toString()); - - synchronized (this) { - this.notify(); - } - } - - @Override - public String toString() { - return super.toString() + " executed=" + executed.get() + "; count=" + - executionCount.get() + ";"; - } - - public long getExecutionCount() { - return executionCount.get(); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java deleted file mode 100644 index 6f4ca42..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate; - -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.api.resource.Application; -import org.apache.slider.providers.PlacementPolicy; -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; -import org.apache.slider.server.appmaster.model.mock.MockRoles; -import org.apache.slider.server.appmaster.state.RoleStatus; - -import static org.apache.slider.api.ResourceKeys.COMPONENT_PLACEMENT_POLICY; - -/** - * Class for basis of Anti-affine placement tests; sets up role2 - * for anti-affinity. - */ -public class BaseMockAppStateAATest extends BaseMockAppStateTest - implements MockRoles { - - /** Role status for the base AA role. */ - private RoleStatus aaRole; - - /** Role status for the AA role requiring a node with the gpu label. */ - private RoleStatus gpuRole; - - @Override - public Application buildApplication() { - Application application = factory.newApplication(0, 0, 0) - .name(getValidTestName()); - application.getComponent(ROLE1).getConfiguration().setProperty( - COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy - .ANTI_AFFINITY_REQUIRED)); - application.getComponent(ROLE1).getConfiguration().setProperty( - ResourceKeys.YARN_LABEL_EXPRESSION, LABEL_GPU); - application.getComponent(ROLE2).getConfiguration().setProperty( - COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy - .ANTI_AFFINITY_REQUIRED)); - return application; - } - - - @Override - public void setup() throws Exception { - super.setup(); - aaRole = lookupRole(ROLE2); - gpuRole = lookupRole(ROLE1); - } - - protected RoleStatus getAaRole() { - return aaRole; - } - - protected RoleStatus getGpuRole() { - return gpuRole; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAOvercapacity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAOvercapacity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAOvercapacity.java deleted file mode 100644 index e0f7c1f..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAOvercapacity.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate; - -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.slider.core.main.LauncherExitCodes; -import org.apache.slider.server.appmaster.model.mock.MockRoles; -import org.apache.slider.server.appmaster.model.mock.MockYarnEngine; -import org.apache.slider.server.appmaster.operations.AbstractRMOperation; -import org.apache.slider.server.appmaster.state.AppState; -import org.apache.slider.server.appmaster.state.NodeInstance; -import org.apache.slider.server.appmaster.state.NodeMap; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.state.RoleStatus; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -/** - * Test Anti-affine placement with a cluster of size 1. - */ -public class TestMockAppStateAAOvercapacity extends BaseMockAppStateAATest - implements MockRoles { - - private static final int NODES = 1; - - @Override - public MockYarnEngine createYarnEngine() { - return new MockYarnEngine(NODES, 1); - } - - void assertAllContainersAA() { - assertAllContainersAA(getAaRole().getKey()); - } - - /** - * - * @throws Throwable - */ - //@Test - public void testOvercapacityRecovery() throws Throwable { - RoleStatus aaRole = getAaRole(); - - describe("Ask for 1 more than the no of available nodes;" + - "verify the state. kill the allocated container and review"); - //more than expected - int desired = 3; - aaRole.setDesired(desired); - assertTrue(appState.getRoleHistory().canPlaceAANodes()); - - //first request - List<AbstractRMOperation> operations = - appState.reviewRequestAndReleaseNodes(); - assertTrue(aaRole.isAARequestOutstanding()); - assertEquals(1, aaRole.getRequested()); - assertEquals(desired - 1, aaRole.getAAPending()); - List<AbstractRMOperation> operationsOut = new ArrayList<>(); - // allocate and re-submit - List<RoleInstance> instances = submitOperations(operations, - EMPTY_ID_LIST, operationsOut); - assertEquals(1, instances.size()); - assertAllContainersAA(); - - // expect an outstanding AA request to be unsatisfied - assertTrue(aaRole.getRunning() < aaRole.getDesired()); - assertEquals(0, aaRole.getRequested()); - assertFalse(aaRole.isAARequestOutstanding()); - assertEquals(desired - 1, aaRole.getAAPending()); - List<Container> allocatedContainers = engine.execute(operations, - EMPTY_ID_LIST); - assertEquals(0, allocatedContainers.size()); - - // now lets trigger a failure - NodeMap nodemap = cloneNodemap(); - assertEquals(1, nodemap.size()); - - RoleInstance instance = instances.get(0); - ContainerId cid = instance.getContainerId(); - - AppState.NodeCompletionResult result = appState.onCompletedContainer( - containerStatus(cid, LauncherExitCodes.EXIT_TASK_LAUNCH_FAILURE)); - assertTrue(result.containerFailed); - - assertEquals(1, aaRole.getFailed()); - assertEquals(0, aaRole.getRunning()); - List<NodeInstance> availablePlacements = appState.getRoleHistory() - .findNodeForNewAAInstance(aaRole); - assertEquals(1, availablePlacements.size()); - describe("expecting a successful review with available placements of " + - availablePlacements); - operations = appState.reviewRequestAndReleaseNodes(); - assertEquals(1, operations.size()); - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
