YARN-5513. Move Java only tests from slider develop to yarn-native-services. Contributed by Gour Saha
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a4beaeb8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a4beaeb8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a4beaeb8 Branch: refs/heads/yarn-native-services Commit: a4beaeb897721f2dfc6540b6bd3073b026b764fd Parents: 0437f8a Author: Jian He <jia...@apache.org> Authored: Wed Aug 17 00:42:24 2016 +0800 Committer: Jian He <jia...@apache.org> Committed: Wed Dec 7 13:00:06 2016 -0800 ---------------------------------------------------------------------- .../dev-support/findbugs-exclude.xml | 20 + .../hadoop-yarn-slider-core/pom.xml | 22 + .../slider/common/tools/TestSliderUtils.java | 159 ++++ .../core/launch/TestAppMasterLauncher.java | 157 ++++ .../TestAppMasterLauncherWithAmReset.java | 92 ++ .../TestPublishedConfigurationOutputter.java | 222 +++++ .../agent/TestAgentClientProvider.java | 77 ++ .../agent/TestAgentLaunchParameter.java | 76 ++ .../slider/providers/agent/TestAgentUtils.java | 94 ++ .../agent/TestAppDefinitionPersister.java | 264 ++++++ .../agent/TestComponentTagProvider.java | 115 +++ .../slider/providers/agent/TestState.java | 33 + .../application/metadata/TestConfigParser.java | 107 +++ .../metadata/TestMetainfoParser.java | 177 ++++ .../appmaster/TestServiceRecordAttributes.java | 68 ++ .../publisher/TestAgentProviderService.java | 60 ++ .../publisher/TestSliderProviderFactory.java | 40 + .../server/servicemonitor/TestPortProbe.java | 37 + .../security/TestCertificateManager.java | 540 +++++++++++ .../TestMultiThreadedStoreGeneration.java | 156 ++++ .../server/services/workflow/MockService.java | 80 ++ .../workflow/ParentWorkflowTestBase.java | 70 ++ .../workflow/ProcessCommandFactory.java | 96 ++ .../services/workflow/SimpleRunnable.java | 46 + .../workflow/TestWorkflowClosingService.java | 116 +++ .../workflow/TestWorkflowCompositeService.java | 113 +++ .../workflow/TestWorkflowExecutorService.java | 66 ++ .../workflow/TestWorkflowRpcService.java | 107 +++ .../workflow/TestWorkflowSequenceService.java | 151 ++++ .../TestWorkflowServiceTerminatingRunnable.java | 64 ++ .../workflow/WorkflowServiceTestBase.java | 139 +++ .../apache/slider/test/ContractTestUtils.java | 901 +++++++++++++++++++ .../slider/test/MiniZooKeeperCluster.java | 395 ++++++++ .../org/apache/slider/tools/TestUtility.java | 181 ++++ .../slider/common/tools/test/metainfo.txt | 16 + .../slider/common/tools/test/metainfo.xml | 98 ++ .../slider/common/tools/test/someOtherFile.txt | 16 + .../slider/common/tools/test/someOtherFile.xml | 17 + .../agent/application/metadata/metainfo.xml | 180 ++++ 39 files changed, 5368 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4beaeb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml new file mode 100644 index 0000000..b89146a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml @@ -0,0 +1,20 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<FindBugsFilter> + +</FindBugsFilter> http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4beaeb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml index 591a5ca..d778f44 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml @@ -129,6 +129,13 @@ <dependency> <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> </dependency> @@ -268,6 +275,20 @@ </dependency> <dependency> + <groupId>org.easymock</groupId> + <artifactId>easymock</artifactId> + <version>3.1</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-api-easymock</artifactId> + <version>1.5</version> + <scope>test</scope> + </dependency> + + <dependency> <groupId>org.mortbay.jetty</groupId> <artifactId>jetty</artifactId> </dependency> @@ -292,6 +313,7 @@ <groupId>org.codehaus.jettison</groupId> <artifactId>jettison</artifactId> </dependency> + <dependency> <groupId>org.mortbay.jetty</groupId> <artifactId>jetty-sslengine</artifactId> http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4beaeb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java new file mode 100644 index 0000000..20e72c0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.common.tools; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationReportPBImpl; +import org.apache.slider.tools.TestUtility; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +/** Test slider util methods. */ +public class TestSliderUtils { + protected static final Logger log = + LoggerFactory.getLogger(TestSliderUtils.class); + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void testGetMetaInfoStreamFromZip() throws Exception { + String zipFileName = TestUtility.createAppPackage( + folder, + "testpkg", + "test.zip", + "target/test-classes/org/apache/slider/common/tools/test"); + Configuration configuration = new Configuration(); + FileSystem fs = FileSystem.getLocal(configuration); + log.info("fs working dir is {}", fs.getWorkingDirectory().toString()); + SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration); + + InputStream stream = SliderUtils.getApplicationResourceInputStream( + sliderFileSystem.getFileSystem(), + new Path(zipFileName), + "metainfo.xml"); + Assert.assertTrue(stream != null); + Assert.assertTrue(stream.available() > 0); + } + + @Test + public void testTruncate() { + Assert.assertEquals(SliderUtils.truncate(null, 5), null); + Assert.assertEquals(SliderUtils.truncate("323", -1), "323"); + Assert.assertEquals(SliderUtils.truncate("3232", 5), "3232"); + Assert.assertEquals(SliderUtils.truncate("1234567890", 0), "1234567890"); + Assert.assertEquals(SliderUtils.truncate("123456789012345", 15), "123456789012345"); + Assert.assertEquals(SliderUtils.truncate("123456789012345", 14), "12345678901..."); + Assert.assertEquals(SliderUtils.truncate("1234567890", 1), "1"); + Assert.assertEquals(SliderUtils.truncate("1234567890", 10), "1234567890"); + Assert.assertEquals(SliderUtils.truncate("", 10), ""); + } + + @Test + public void testApplicationReportComparison() { + List<ApplicationReport> instances = getApplicationReports(); + + SliderUtils.sortApplicationsByMostRecent(instances); + + Assert.assertEquals(1000, instances.get(0).getStartTime()); + Assert.assertEquals(1000, instances.get(1).getStartTime()); + Assert.assertEquals(1000, instances.get(2).getStartTime()); + Assert.assertEquals(1000, instances.get(3).getStartTime()); + + instances = getApplicationReports(); + + SliderUtils.sortApplicationReport(instances); + Assert.assertEquals(1000, instances.get(0).getStartTime()); + Assert.assertEquals(1000, instances.get(1).getStartTime()); + Assert.assertEquals(1000, instances.get(2).getStartTime()); + Assert.assertEquals(1000, instances.get(3).getStartTime()); + + Assert.assertTrue(instances.get(0).getYarnApplicationState() == YarnApplicationState.ACCEPTED || + instances.get(0).getYarnApplicationState() == YarnApplicationState.RUNNING); + Assert.assertTrue(instances.get(1).getYarnApplicationState() == YarnApplicationState.ACCEPTED || + instances.get(1).getYarnApplicationState() == YarnApplicationState.RUNNING); + Assert.assertTrue(instances.get(2).getYarnApplicationState() == YarnApplicationState.ACCEPTED || + instances.get(2).getYarnApplicationState() == YarnApplicationState.RUNNING); + Assert.assertTrue(instances.get(3).getYarnApplicationState() == YarnApplicationState.KILLED); + } + + private List<ApplicationReport> getApplicationReports() { + List<ApplicationReport> instances = new ArrayList<ApplicationReport>(); + instances.add(getApplicationReport(1000, 0, "app1", YarnApplicationState.ACCEPTED)); + instances.add(getApplicationReport(900, 998, "app1", YarnApplicationState.KILLED)); + instances.add(getApplicationReport(900, 998, "app2", YarnApplicationState.FAILED)); + instances.add(getApplicationReport(1000, 0, "app2", YarnApplicationState.RUNNING)); + instances.add(getApplicationReport(800, 837, "app3", YarnApplicationState.FINISHED)); + instances.add(getApplicationReport(1000, 0, "app3", YarnApplicationState.RUNNING)); + instances.add(getApplicationReport(900, 998, "app3", YarnApplicationState.KILLED)); + instances.add(getApplicationReport(800, 837, "app4", YarnApplicationState.FINISHED)); + instances.add(getApplicationReport(1000, 1050, "app4", YarnApplicationState.KILLED)); + instances.add(getApplicationReport(900, 998, "app4", YarnApplicationState.FINISHED)); + + Assert.assertEquals("app1", instances.get(0).getApplicationType()); + Assert.assertEquals("app1", instances.get(1).getApplicationType()); + Assert.assertEquals("app2", instances.get(2).getApplicationType()); + Assert.assertEquals("app2", instances.get(3).getApplicationType()); + return instances; + } + + private ApplicationReportPBImpl getApplicationReport(long startTime, + long finishTime, + String name, + YarnApplicationState state) { + ApplicationReportPBImpl ar = new ApplicationReportPBImpl(); + ar.setFinishTime(finishTime); + ar.setStartTime(startTime); + ar.setApplicationType(name); + ar.setYarnApplicationState(state); + return ar; + } + + + @Test + public void testGetHdpVersion() { + String hdpVersion = "2.3.2.0-2766"; + Assert.assertEquals("Version should be empty", null, + SliderUtils.getHdpVersion()); + } + + @Test + public void testIsHdp() { + Assert.assertFalse("Should be false", SliderUtils.isHdp()); + } + + @Test + public void testWrite() throws IOException { + File testWriteFile = folder.newFile("testWrite"); + SliderUtils.write(testWriteFile, "test".getBytes("UTF-8"), true); + Assert.assertTrue(FileUtils.readFileToString(testWriteFile, "UTF-8").equals("test")); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4beaeb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java new file mode 100644 index 0000000..b955931 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java @@ -0,0 +1,157 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.launch; + +import java.lang.reflect.Method; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.LogAggregationContext; +import org.apache.hadoop.yarn.client.api.YarnClientApplication; +import org.apache.slider.api.ResourceKeys; +import org.apache.slider.client.SliderYarnClientImpl; +import org.apache.slider.common.SliderKeys; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class TestAppMasterLauncher { + SliderYarnClientImpl mockYarnClient; + YarnClientApplication yarnClientApp; + ApplicationSubmissionContext appSubmissionContext; + Set<String> tags = Collections.emptySet(); + AppMasterLauncher appMasterLauncher = null; + boolean isOldApi = true; + Method rolledLogsIncludeMethod = null; + Method rolledLogsExcludeMethod = null; + + @Before + public void initialize() throws Exception { + mockYarnClient = EasyMock.createNiceMock(SliderYarnClientImpl.class); + yarnClientApp = EasyMock.createNiceMock(YarnClientApplication.class); + appSubmissionContext = EasyMock + .createNiceMock(ApplicationSubmissionContext.class); + EasyMock.expect(yarnClientApp.getApplicationSubmissionContext()) + .andReturn(appSubmissionContext).once(); + EasyMock.expect(mockYarnClient.createApplication()) + .andReturn(yarnClientApp).once(); + + try { + LogAggregationContext.class.getMethod("newInstance", String.class, + String.class, String.class, String.class); + isOldApi = false; + rolledLogsIncludeMethod = LogAggregationContext.class + .getMethod("getRolledLogsIncludePattern"); + rolledLogsExcludeMethod = LogAggregationContext.class + .getMethod("getRolledLogsExcludePattern"); + } catch (Exception e) { + isOldApi = true; + } + } + + /** + * These tests will probably fail when compiled against hadoop 2.7+. Please + * refer to SLIDER-810. It has been purposely not modified so that it fails + * and that someone needs to modify the code in + * {@code AbstractLauncher#extractLogAggregationContext(Map)}. Comments are + * provided in that method as to what needs to be done. + * + * @throws Exception + */ + @Test + public void testExtractLogAggregationContext() throws Exception { + Map<String, String> options = new HashMap<String, String>(); + options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, + " | slider*.txt |agent.out| |"); + options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, + "command*.json| agent.log* | "); + + EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp); + appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null, + null, mockYarnClient, false, null, options, tags, null); + + // Verify the include/exclude patterns + String expectedInclude = "slider*.txt|agent.out"; + String expectedExclude = "command*.json|agent.log*"; + assertPatterns(expectedInclude, expectedExclude); + + EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp); + + } + + @Test + public void testExtractLogAggregationContextEmptyIncludePattern() + throws Exception { + Map<String, String> options = new HashMap<String, String>(); + options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, " "); + options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, + "command*.json| agent.log* | "); + + EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp); + appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null, + null, mockYarnClient, false, null, options, tags, null); + + // Verify the include/exclude patterns + String expectedInclude = isOldApi ? "" : ".*"; + String expectedExclude = "command*.json|agent.log*"; + assertPatterns(expectedInclude, expectedExclude); + + EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp); + } + + @Test + public void testExtractLogAggregationContextEmptyIncludeAndExcludePattern() + throws Exception { + Map<String, String> options = new HashMap<String, String>(); + options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, ""); + options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, " "); + + EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp); + appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null, + null, mockYarnClient, false, null, options, tags, null); + + // Verify the include/exclude patterns + String expectedInclude = isOldApi ? "" : ".*"; + String expectedExclude = ""; + assertPatterns(expectedInclude, expectedExclude); + + EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp); + } + + private void assertPatterns(String expectedIncludePattern, + String expectedExcludePattern) throws Exception { + if (isOldApi) { + Assert.assertEquals(expectedIncludePattern, + appMasterLauncher.logAggregationContext.getIncludePattern()); + Assert.assertEquals(expectedExcludePattern, + appMasterLauncher.logAggregationContext.getExcludePattern()); + } else { + Assert.assertEquals(expectedIncludePattern, + (String) rolledLogsIncludeMethod + .invoke(appMasterLauncher.logAggregationContext)); + Assert.assertEquals(expectedExcludePattern, + (String) rolledLogsExcludeMethod + .invoke(appMasterLauncher.logAggregationContext)); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4beaeb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java new file mode 100644 index 0000000..a8f6b26 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.launch; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.client.api.YarnClientApplication; +import org.apache.hadoop.yarn.util.Records; +import org.apache.slider.api.ResourceKeys; +import org.apache.slider.client.SliderYarnClientImpl; +import org.apache.slider.common.SliderKeys; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class TestAppMasterLauncherWithAmReset { + SliderYarnClientImpl mockYarnClient; + YarnClientApplication yarnClientApp; + ApplicationSubmissionContext appSubmissionContext; + GetNewApplicationResponse newApp; + Set<String> tags = Collections.emptySet(); + AppMasterLauncher appMasterLauncher = null; + boolean isOldApi = true; + + @Before + public void initialize() throws Exception { + mockYarnClient = EasyMock.createNiceMock(SliderYarnClientImpl.class); + yarnClientApp = EasyMock.createNiceMock(YarnClientApplication.class); + newApp = EasyMock.createNiceMock(GetNewApplicationResponse.class); + EasyMock.expect(mockYarnClient.createApplication()) + .andReturn(new YarnClientApplication(newApp, + Records.newRecord(ApplicationSubmissionContext.class))); + } + + @Test + public void testExtractYarnResourceManagerAmRetryCountWindowMs() throws + Exception { + Map<String, String> options = new HashMap<String, String>(); + final String expectedInterval = Integer.toString (120000); + options.put(ResourceKeys.YARN_RESOURCEMANAGER_AM_RETRY_COUNT_WINDOW_MS, + expectedInterval); + EasyMock.replay(mockYarnClient, yarnClientApp); + + appMasterLauncher = new AppMasterLauncher("am1", SliderKeys.APP_TYPE, null, + null, mockYarnClient, false, null, options, tags, null); + + ApplicationSubmissionContext ctx = appMasterLauncher.application + .getApplicationSubmissionContext(); + String retryIntervalWindow = Long.toString(ctx + .getAttemptFailuresValidityInterval()); + Assert.assertEquals(expectedInterval, retryIntervalWindow); + } + + @Test + public void testExtractYarnResourceManagerAmRetryCountWindowMsDefaultValue() + throws Exception { + Map<String, String> options = new HashMap<String, String>(); + EasyMock.replay(mockYarnClient, yarnClientApp); + + appMasterLauncher = new AppMasterLauncher("am1", SliderKeys.APP_TYPE, null, + null, mockYarnClient, false, null, options, tags, null); + + ApplicationSubmissionContext ctx = appMasterLauncher.application + .getApplicationSubmissionContext(); + long retryIntervalWindow = ctx.getAttemptFailuresValidityInterval(); + Assert.assertEquals(ResourceKeys.DEFAULT_AM_RETRY_COUNT_WINDOW_MS, + retryIntervalWindow); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4beaeb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java new file mode 100644 index 0000000..3706182 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.core.registry.docstore; + +import com.google.common.base.Charsets; +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.fs.Path; +import org.apache.slider.common.tools.SliderFileSystem; +import org.codehaus.jackson.map.ObjectMapper; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.powermock.api.easymock.PowerMock; +import org.yaml.snakeyaml.Yaml; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.expect; +import static org.mockito.Matchers.anyString; +import static org.powermock.api.easymock.PowerMock.createNiceMock; + +public class TestPublishedConfigurationOutputter { + private static HashMap<String, String> config = new HashMap<>(); + + @Rule + public TemporaryFolder tmpDir = new TemporaryFolder(); + + @Before + public void setup() { + config.put("key1", "val1"); + } + + @Test + public void testJson() throws IOException { + PublishedConfigurationOutputter configurationOutputter = + PublishedConfigurationOutputter.createOutputter(ConfigFormat.JSON, + new PublishedConfiguration("description", + config.entrySet())); + + String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)", + ""); + assert "{\"key1\":\"val1\"}".equals(output); + + File file = tmpDir.newFile(); + configurationOutputter.save(file); + + ObjectMapper mapper = new ObjectMapper(); + @SuppressWarnings("unchecked") + Map<String, String> read = mapper.readValue(file, Map.class); + assert 1 == read.size(); + assert "val1".equals(read.get("key1")); + } + + @Test + public void testXml() throws IOException { + PublishedConfigurationOutputter configurationOutputter = + PublishedConfigurationOutputter.createOutputter(ConfigFormat.XML, + new PublishedConfiguration("description", + config.entrySet())); + + String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)", + ""); + assert output.contains( + "<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>"); + + File file = tmpDir.newFile(); + configurationOutputter.save(file); + + assert FileUtils.readFileToString(file, Charsets.UTF_8) + .replaceAll("( |\\r|\\n)", "") + .contains( + "<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>"); + } + + @Test + public void testHadoopXml() throws IOException { + PublishedConfigurationOutputter configurationOutputter = + PublishedConfigurationOutputter.createOutputter(ConfigFormat.HADOOP_XML, + new PublishedConfiguration("description", + config.entrySet())); + + String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)", + ""); + assert output.contains("<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>"); + + File file = tmpDir.newFile(); + configurationOutputter.save(file); + + assert FileUtils.readFileToString(file, Charsets.UTF_8) + .replaceAll("( |\\r|\\n)", "") + .contains( "<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>"); + } + + @Test + public void testProperties() throws IOException { + PublishedConfigurationOutputter configurationOutputter = + PublishedConfigurationOutputter.createOutputter(ConfigFormat.PROPERTIES, + new PublishedConfiguration("description", + config.entrySet())); + + String output = configurationOutputter.asString(); + assert output.contains("key1=val1"); + + File file = tmpDir.newFile(); + configurationOutputter.save(file); + + Properties properties = new Properties(); + FileInputStream fis = null; + try { + fis = new FileInputStream(file); + properties.load(fis); + } finally { + if (fis != null) { + fis.close(); + } + } + assert 1 == properties.size(); + assert "val1".equals(properties.getProperty("key1")); + } + + @Test + public void testYaml() throws IOException { + PublishedConfigurationOutputter configurationOutputter = + PublishedConfigurationOutputter.createOutputter(ConfigFormat.YAML, + new PublishedConfiguration("description", + config.entrySet())); + + String output = configurationOutputter.asString().replaceAll("(\\r|\\n)", + ""); + assert "key1: val1".equals(output); + + File file = tmpDir.newFile(); + configurationOutputter.save(file); + + Yaml yaml = new Yaml(); + FileInputStream fis = null; + Map<String, String> read; + try { + fis = new FileInputStream(file); + read = (Map<String, String>) yaml.load(fis); + } finally { + if (fis != null) { + fis.close(); + } + } + assert 1 == read.size(); + assert "val1".equals(read.get("key1")); + } + + @Test + public void testEnv() throws IOException { + HashMap<String, String> envConfig = new HashMap<>(config); + envConfig.put("content", "content {{key1}} "); + + PublishedConfigurationOutputter configurationOutputter = + PublishedConfigurationOutputter.createOutputter(ConfigFormat.ENV, + new PublishedConfiguration("description", + envConfig.entrySet())); + + String output = configurationOutputter.asString(); + assert "content val1 ".equals(output); + + File file = tmpDir.newFile(); + configurationOutputter.save(file); + + assert "content val1 ".equals(FileUtils.readFileToString(file, + Charsets.UTF_8)); + } + + @Test + public void testTemplate1() throws IOException { + HashMap<String, String> templateConfig = new HashMap<>(config); + templateConfig.put(ConfigUtils.TEMPLATE_FILE, "templateFileName"); + + SliderFileSystem fileSystem = createNiceMock(SliderFileSystem.class); + expect(fileSystem.buildResourcePath(anyString())).andReturn(new Path("path")).anyTimes(); + expect(fileSystem.isFile(anyObject(Path.class))).andReturn(true).anyTimes(); + expect(fileSystem.cat(anyObject(Path.class))).andReturn("content {{key1}}\n more ${key1} content").anyTimes(); + + PowerMock.replay(fileSystem); + + ConfigUtils.prepConfigForTemplateOutputter(ConfigFormat.TEMPLATE, + templateConfig, fileSystem, "clusterName", null); + PublishedConfigurationOutputter configurationOutputter = + PublishedConfigurationOutputter.createOutputter(ConfigFormat.TEMPLATE, + new PublishedConfiguration("description", + templateConfig.entrySet())); + + String output = configurationOutputter.asString(); + assert "content val1\n more val1 content".equals(output); + + File file = tmpDir.newFile(); + configurationOutputter.save(file); + + PowerMock.verify(fileSystem); + + assert "content val1\n more val1 content".equals( + FileUtils.readFileToString(file, Charsets.UTF_8)); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4beaeb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java new file mode 100644 index 0000000..0bea8fa --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.core.exceptions.BadConfigException; +import org.apache.slider.tools.TestUtility; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Set; + +/** + * + */ +public class TestAgentClientProvider { + protected static final Logger log = + LoggerFactory.getLogger(TestAgentClientProvider.class); + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void testGetApplicationTags() throws Exception { + Configuration configuration = new Configuration(); + FileSystem fs = FileSystem.getLocal(configuration); + SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration); + + AgentClientProvider provider = new AgentClientProvider(null); + String zipFileName = TestUtility.createAppPackage( + folder, + "testpkg", + "test.zip", + "target/test-classes/org/apache/slider/common/tools/test"); + Set<String> tags = provider.getApplicationTags(sliderFileSystem, zipFileName); + assert tags != null; + assert !tags.isEmpty(); + assert tags.contains("Name: STORM"); + assert tags.contains("Description: Apache Hadoop Stream processing framework"); + assert tags.contains("Version: 0.9.1.2.1"); + + } + + @Test + public void testValidateInstanceDefinition() throws Exception { + AgentClientProvider provider = new AgentClientProvider(null); + AggregateConf instanceDefinition = new AggregateConf(); + + try { + provider.validateInstanceDefinition(instanceDefinition, null); + Assert.assertFalse("Should fail with BadConfigException", true); + } catch (BadConfigException e) { + log.info(e.toString()); + Assert.assertTrue(e.getMessage().contains("Application definition must be provided")); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4beaeb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentLaunchParameter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentLaunchParameter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentLaunchParameter.java new file mode 100644 index 0000000..ec62b54 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentLaunchParameter.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent; + +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * + */ +public class TestAgentLaunchParameter { + protected static final Logger log = + LoggerFactory.getLogger(TestAgentLaunchParameter.class); + + @Test + public void testTestAgentLaunchParameter() throws Exception { + AgentLaunchParameter alp = new AgentLaunchParameter(""); + Assert.assertEquals("", alp.getNextLaunchParameter("abc")); + Assert.assertEquals("", alp.getNextLaunchParameter("HBASE_MASTER")); + + alp = new AgentLaunchParameter("a:1:2:3|b:5:6:NONE"); + Assert.assertEquals("1", alp.getNextLaunchParameter("a")); + Assert.assertEquals("2", alp.getNextLaunchParameter("a")); + Assert.assertEquals("3", alp.getNextLaunchParameter("a")); + Assert.assertEquals("3", alp.getNextLaunchParameter("a")); + + Assert.assertEquals("5", alp.getNextLaunchParameter("b")); + Assert.assertEquals("6", alp.getNextLaunchParameter("b")); + Assert.assertEquals("", alp.getNextLaunchParameter("b")); + Assert.assertEquals("", alp.getNextLaunchParameter("b")); + Assert.assertEquals("", alp.getNextLaunchParameter("c")); + + alp = new AgentLaunchParameter("|a:1:3|b::5:NONE:"); + Assert.assertEquals("1", alp.getNextLaunchParameter("a")); + Assert.assertEquals("3", alp.getNextLaunchParameter("a")); + Assert.assertEquals("3", alp.getNextLaunchParameter("a")); + + Assert.assertEquals("", alp.getNextLaunchParameter("b")); + Assert.assertEquals("5", alp.getNextLaunchParameter("b")); + Assert.assertEquals("", alp.getNextLaunchParameter("b")); + Assert.assertEquals("", alp.getNextLaunchParameter("b")); + + alp = new AgentLaunchParameter("|:"); + Assert.assertEquals("", alp.getNextLaunchParameter("b")); + Assert.assertEquals("", alp.getNextLaunchParameter("a")); + + alp = new AgentLaunchParameter("HBASE_MASTER:a,b:DO_NOT_REGISTER:"); + Assert.assertEquals("a,b", alp.getNextLaunchParameter("HBASE_MASTER")); + Assert.assertEquals("DO_NOT_REGISTER", alp.getNextLaunchParameter("HBASE_MASTER")); + Assert.assertEquals("DO_NOT_REGISTER", alp.getNextLaunchParameter("HBASE_MASTER")); + + alp = new AgentLaunchParameter("HBASE_MASTER:a,b:DO_NOT_REGISTER::c:::"); + Assert.assertEquals("a,b", alp.getNextLaunchParameter("HBASE_MASTER")); + Assert.assertEquals("DO_NOT_REGISTER", alp.getNextLaunchParameter("HBASE_MASTER")); + Assert.assertEquals("", alp.getNextLaunchParameter("HBASE_MASTER")); + Assert.assertEquals("c", alp.getNextLaunchParameter("HBASE_MASTER")); + Assert.assertEquals("c", alp.getNextLaunchParameter("HBASE_MASTER")); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4beaeb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java new file mode 100644 index 0000000..5e1dc7f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.providers.agent; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.providers.agent.application.metadata.Metainfo; +import org.apache.slider.tools.TestUtility; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestAgentUtils { + protected static final Logger log = + LoggerFactory.getLogger(TestAgentUtils.class); + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + private static final String metainfo_str = "<metainfo>\n" + + " <schemaVersion>2.0</schemaVersion>\n" + + " <application>\n" + + " <name>MYTESTAPPLICATION</name>\n" + + " <comment>\n" + + " My Test Application\n" + + " </comment>\n" + + " <version>1.0</version>\n" + + " <type>YARN-APP</type>\n" + + " <components>\n" + + " <component>\n" + + " <name>REST</name>\n" + + " <category>MASTER</category>\n" + + " <commandScript>\n" + + " <script>scripts/rest.py</script>\n" + + " <scriptType>PYTHON</scriptType>\n" + + " <timeout>600</timeout>\n" + + " </commandScript>\n" + + " </component>\n" + + " </components>\n" + + " </application>\n" + + "</metainfo>"; + + @Test + public void testGetApplicationMetainfo() throws Exception { + String zipFileName = TestUtility.createAppPackage( + folder, + "testpkg", + "test.zip", + "target/test-classes/org/apache/slider/common/tools/test"); + Configuration configuration = new Configuration(); + FileSystem fs = FileSystem.getLocal(configuration); + log.info("fs working dir is {}", fs.getWorkingDirectory().toString()); + SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration); + + // Without accompany metainfo file, read metainfo from the zip file + Metainfo metainfo = AgentUtils.getApplicationMetainfo( + sliderFileSystem, zipFileName, false); + Assert.assertNotNull(metainfo.getApplication()); + Assert.assertEquals("STORM", metainfo.getApplication().getName()); + + // With accompany metainfo file, read metainfo from the accompany file + String acompanyFileName = zipFileName + ".metainfo.xml"; + File f = new File(acompanyFileName); + try (BufferedWriter writer = new BufferedWriter(new FileWriter(f))) { + writer.write(metainfo_str); + } + metainfo = AgentUtils.getApplicationMetainfo( + sliderFileSystem, zipFileName, false); + Assert.assertNotNull(metainfo.getApplication()); + Assert.assertEquals("MYTESTAPPLICATION", metainfo.getApplication().getName()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4beaeb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java new file mode 100644 index 0000000..dedf4f6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent; + +import com.google.common.io.Files; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.slider.common.params.ActionCreateArgs; +import org.apache.slider.common.params.AddonArgsDelegate; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.core.conf.ConfTree; +import org.apache.slider.core.conf.ConfTreeOperations; +import org.apache.slider.core.exceptions.BadConfigException; +import org.apache.slider.core.persist.AppDefinitionPersister; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; + +/** + * + */ +public class TestAppDefinitionPersister { + protected static final Logger log = + LoggerFactory.getLogger(TestAppDefinitionPersister.class); + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + /** + * @BeforeClass public static void initialize() { BasicConfigurator.resetConfiguration(); + * BasicConfigurator.configure(); }* + */ + + + @Test + public void testAppDefinitionPersister() throws Exception { + Configuration configuration = new Configuration(); + FileSystem fs = FileSystem.getLocal(configuration); + log.info("fs working dir is {}", fs.getWorkingDirectory().toString()); + SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration); + + AppDefinitionPersister adp = new AppDefinitionPersister(sliderFileSystem); + String clustername = "c1"; + ActionCreateArgs buildInfo = new ActionCreateArgs(); + buildInfo.appMetaInfo = null; + buildInfo.appDef = null; + buildInfo.addonDelegate = new AddonArgsDelegate(); + + // nothing to do + adp.processSuppliedDefinitions(clustername, buildInfo, null); + adp.persistPackages(); + List<AppDefinitionPersister.AppDefinition> appDefinitions = adp.getAppDefinitions(); + Assert.assertTrue(appDefinitions.size() == 0); + + ConfTree ct = new ConfTree(); + ConfTreeOperations appConf = new ConfTreeOperations(ct); + final File tempDir = Files.createTempDir(); + final File metainfo = new File(tempDir, "metainfo.json"); + + // unreadable metainfo + buildInfo.appMetaInfo = metainfo; + + try { + adp.processSuppliedDefinitions(clustername, buildInfo, appConf); + } catch (BadConfigException bce) { + log.info(bce.getMessage()); + Assert.assertTrue(bce.getMessage().contains( + "Path specified with " + + "--metainfo either cannot be read or is not a file")); + } + + try (PrintWriter writer = new PrintWriter(metainfo.getAbsolutePath(), "UTF-8")) { + writer.println("{"); + writer.println("}"); + } + buildInfo.appDef = metainfo; + + try { + adp.processSuppliedDefinitions(clustername, buildInfo, appConf); + } catch (BadConfigException bce) { + log.info(bce.getMessage()); + Assert.assertTrue(bce.getMessage().contains( + "Both --metainfo and --appdef cannot be specified")); + } + + // both --metainfojson and --appdef cannot be specified + buildInfo.appMetaInfo = null; + buildInfo.appMetaInfoJson = "{}"; + try { + adp.processSuppliedDefinitions(clustername, buildInfo, appConf); + } catch (BadConfigException bce) { + log.info(bce.getMessage()); + Assert.assertTrue(bce.getMessage().contains( + "Both --metainfojson and --appdef cannot be specified")); + } + + buildInfo.appDef = null; + + buildInfo.appMetaInfoJson = ""; + try { + adp.processSuppliedDefinitions(clustername, buildInfo, appConf); + } catch (BadConfigException bce) { + log.info(bce.getMessage()); + Assert.assertTrue(bce.getMessage().contains( + "Empty string specified with --metainfojson")); + } + buildInfo.appMetaInfo = metainfo; + + // both --metainfo and --metainfojson cannot be specified + buildInfo.appMetaInfoJson = "{}"; + try { + adp.processSuppliedDefinitions(clustername, buildInfo, appConf); + } catch (BadConfigException bce) { + log.info(bce.getMessage()); + Assert.assertTrue(bce.getMessage().contains( + "Both --metainfo and --metainfojson cannot be specified")); + } + buildInfo.appMetaInfoJson = null; + + appConf.getGlobalOptions().set(AgentKeys.APP_DEF, metainfo.getAbsolutePath()); + + try { + adp.processSuppliedDefinitions(clustername, buildInfo, appConf); + } catch (BadConfigException bce) { + log.info(bce.getMessage()); + Assert.assertTrue(bce.getMessage().contains( + "application.def cannot " + + "not be set if --metainfo is specified in the cmd line")); + } + + appConf.getGlobalOptions().remove(AgentKeys.APP_DEF); + + adp.processSuppliedDefinitions(clustername, buildInfo, appConf); + appDefinitions = adp.getAppDefinitions(); + Assert.assertTrue(appDefinitions.size() == 1); + Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip")); + log.info(appDefinitions.get(0).toString()); + Assert.assertTrue(appDefinitions.get(0).appDefPkgOrFolder.toString().endsWith("default")); + Assert.assertTrue(appDefinitions.get(0).targetFolderInFs.toString().contains("cluster/c1/appdef")); + Assert.assertEquals("appPkg.zip", appDefinitions.get(0).pkgName); + + buildInfo.appDef = tempDir; + buildInfo.appMetaInfo = null; + + appConf.getGlobalOptions().set(AgentKeys.APP_DEF, metainfo.getAbsolutePath()); + + try { + adp.processSuppliedDefinitions(clustername, buildInfo, appConf); + } catch (BadConfigException bce) { + log.info(bce.getMessage()); + Assert.assertTrue(bce.getMessage().contains("application.def must not be set if --appdef is provided")); + } + + adp.getAppDefinitions().clear(); + appConf.getGlobalOptions().remove(AgentKeys.APP_DEF); + adp.processSuppliedDefinitions(clustername, buildInfo, appConf); + appDefinitions = adp.getAppDefinitions(); + Assert.assertTrue(appDefinitions.size() == 1); + Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip")); + log.info(appDefinitions.get(0).toString()); + Assert.assertTrue(appDefinitions.get(0).appDefPkgOrFolder.toString().endsWith(tempDir.toString())); + Assert.assertTrue(appDefinitions.get(0).targetFolderInFs.toString().contains("cluster/c1/appdef")); + Assert.assertEquals("appPkg.zip", appDefinitions.get(0).pkgName); + + adp.getAppDefinitions().clear(); + buildInfo.appDef = null; + buildInfo.appMetaInfo = null; + appConf.getGlobalOptions().remove(AgentKeys.APP_DEF); + + ArrayList<String> list = new ArrayList<String>() {{ + add("addon1"); + add(""); + add("addon2"); + add(metainfo.getAbsolutePath()); + }}; + + buildInfo.addonDelegate.addonTuples = list; + try { + adp.processSuppliedDefinitions(clustername, buildInfo, appConf); + } catch (BadConfigException bce) { + log.info(bce.getMessage()); + Assert.assertTrue(bce.getMessage().contains("addon package can only be specified if main app package is specified")); + } + + buildInfo.appMetaInfo = metainfo; + + try { + adp.processSuppliedDefinitions(clustername, buildInfo, appConf); + } catch (BadConfigException bce) { + log.info(bce.getMessage()); + Assert.assertTrue(bce.getMessage().contains("Invalid path for addon package addon1")); + } + + appConf.getGlobalOptions().remove(AgentKeys.APP_DEF); + + list = new ArrayList<String>() {{ + add("addon1"); + add(tempDir.getAbsolutePath()); + add("addon2"); + add(metainfo.getAbsolutePath()); + }}; + + buildInfo.addonDelegate.addonTuples = list; + adp.getAppDefinitions().clear(); + + adp.processSuppliedDefinitions(clustername, buildInfo, appConf); + appDefinitions = adp.getAppDefinitions(); + + Assert.assertTrue(appDefinitions.size() == 3); + Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip")); + Assert.assertTrue(appConf.getGlobalOptions().get("application.addon.addon1").contains( + "addons/addon1/addon_addon1.zip")); + Assert.assertTrue(appConf.getGlobalOptions().get("application.addon.addon2").contains( + "addons/addon2/addon_addon2.zip")); + log.info(appConf.getGlobalOptions().get("application.addons")); + Assert.assertTrue(appConf.getGlobalOptions().get("application.addons").contains( + "application.addon.addon2,application.addon.addon1") + || appConf.getGlobalOptions().get("application.addons").contains( + "application.addon.addon1,application.addon.addon2")); + int seen = 0; + for (AppDefinitionPersister.AppDefinition adp_ad : appDefinitions) { + if (adp_ad.pkgName.equals("appPkg.zip")) { + log.info(adp_ad.toString()); + Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith("default")); + Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("cluster/c1/appdef")); + seen++; + } + if (adp_ad.pkgName.equals("addon_addon1.zip")) { + log.info(adp_ad.toString()); + Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith(tempDir.toString())); + Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("addons/addon1")); + seen++; + } + if (adp_ad.pkgName.equals("addon_addon2.zip")) { + log.info(adp_ad.toString()); + Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith("metainfo.json")); + Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("addons/addon2")); + seen++; + } + } + Assert.assertEquals(3, seen); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4beaeb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentTagProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentTagProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentTagProvider.java new file mode 100644 index 0000000..7b38ee3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentTagProvider.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.providers.agent; + +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class TestComponentTagProvider { + protected static final Logger log = + LoggerFactory.getLogger(TestComponentTagProvider.class); + + @Test + public void testTagProvider() throws Exception { + ComponentTagProvider ctp = new ComponentTagProvider(); + Assert.assertEquals("", ctp.getTag(null, null)); + Assert.assertEquals("", ctp.getTag(null, "cid")); + Assert.assertEquals("", ctp.getTag("comp1", null)); + + Assert.assertEquals("1", ctp.getTag("comp1", "cid1")); + Assert.assertEquals("2", ctp.getTag("comp1", "cid2")); + Assert.assertEquals("3", ctp.getTag("comp1", "cid3")); + ctp.releaseTag("comp1", "cid2"); + Assert.assertEquals("2", ctp.getTag("comp1", "cid22")); + + ctp.releaseTag("comp1", "cid4"); + ctp.recordAssignedTag("comp1", "cid5", "5"); + Assert.assertEquals("4", ctp.getTag("comp1", "cid4")); + Assert.assertEquals("4", ctp.getTag("comp1", "cid4")); + Assert.assertEquals("6", ctp.getTag("comp1", "cid6")); + + ctp.recordAssignedTag("comp1", "cid55", "5"); + Assert.assertEquals("5", ctp.getTag("comp1", "cid55")); + + ctp.recordAssignedTag("comp2", "cidb3", "3"); + Assert.assertEquals("1", ctp.getTag("comp2", "cidb1")); + Assert.assertEquals("2", ctp.getTag("comp2", "cidb2")); + Assert.assertEquals("4", ctp.getTag("comp2", "cidb4")); + + ctp.recordAssignedTag("comp2", "cidb5", "six"); + ctp.recordAssignedTag("comp2", "cidb5", "-55"); + ctp.recordAssignedTag("comp2", "cidb5", "tags"); + ctp.recordAssignedTag("comp2", "cidb5", null); + ctp.recordAssignedTag("comp2", "cidb5", ""); + ctp.recordAssignedTag("comp2", "cidb5", "5"); + Assert.assertEquals("6", ctp.getTag("comp2", "cidb6")); + + ctp.recordAssignedTag("comp2", null, "5"); + ctp.recordAssignedTag(null, null, "5"); + ctp.releaseTag("comp1", null); + ctp.releaseTag(null, "cid4"); + ctp.releaseTag(null, null); + } + + @Test + public void testTagProviderWithThread() throws Exception { + ComponentTagProvider ctp = new ComponentTagProvider(); + Thread thread = new Thread(new Taggged(ctp)); + Thread thread2 = new Thread(new Taggged(ctp)); + Thread thread3 = new Thread(new Taggged(ctp)); + thread.start(); + thread2.start(); + thread3.start(); + ctp.getTag("comp1", "cid50"); + thread.join(); + thread2.join(); + thread3.join(); + Assert.assertEquals("101", ctp.getTag("comp1", "cid101")); + } + + public class Taggged implements Runnable { + private final ComponentTagProvider ctp; + + public Taggged(ComponentTagProvider ctp) { + this.ctp = ctp; + } + + public void run() { + for (int i = 0; i < 100; i++) { + String containerId = "cid" + (i + 1); + this.ctp.getTag("comp1", containerId); + } + for (int i = 0; i < 100; i++) { + String containerId = "cid" + (i + 1); + this.ctp.getTag("comp1", containerId); + } + for (int i = 0; i < 100; i += 2) { + String containerId = "cid" + (i + 1); + this.ctp.releaseTag("comp1", containerId); + } + for (int i = 0; i < 100; i += 2) { + String containerId = "cid" + (i + 1); + this.ctp.getTag("comp1", containerId); + } + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4beaeb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestState.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestState.java new file mode 100644 index 0000000..6a2e5ab5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestState.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent; + +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestState { + protected static final Logger log = LoggerFactory.getLogger(TestState.class); + + @Test + public void testState() throws Exception { + State state = State.STARTED; + Assert.assertEquals(Command.STOP, state.getSupportedCommand(false, true)); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4beaeb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestConfigParser.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestConfigParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestConfigParser.java new file mode 100644 index 0000000..3aa44a1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestConfigParser.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent.application.metadata; + +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; + +/** + * + */ +public class TestConfigParser { + protected static final Logger log = + LoggerFactory.getLogger(TestConfigParser.class); + private static final String config_1_str = "<configuration>\n" + + " <property>\n" + + " <name>security.client.protocol.acl</name>\n" + + " <value>*</value>\n" + + " <description>ACL for HRegionInterface protocol implementations (ie. \n" + + " clients talking to HRegionServers)\n" + + " The ACL is a comma-separated list of user and group names. The user and \n" + + " group list is separated by a blank. For e.g. \"alice,bob users,wheel\". \n" + + " A special value of \"*\" means all users are allowed.</description>\n" + + " </property>\n" + + "\n" + + " <property>\n" + + " <name>security.admin.protocol.acl</name>\n" + + " <value>*</value>\n" + + " <description>ACL for HMasterInterface protocol implementation (ie. \n" + + " clients talking to HMaster for admin operations).\n" + + " The ACL is a comma-separated list of user and group names. The user and \n" + + " group list is separated by a blank. For e.g. \"alice,bob users,wheel\". \n" + + " A special value of \"*\" means all users are allowed.</description>\n" + + " </property>\n" + + "\n" + + " <property>\n" + + " <name>security.masterregion.protocol.acl</name>\n" + + " <value>*</value>\n" + + " <description>ACL for HMasterRegionInterface protocol implementations\n" + + " (for HRegionServers communicating with HMaster)\n" + + " The ACL is a comma-separated list of user and group names. The user and \n" + + " group list is separated by a blank. For e.g. \"alice,bob users,wheel\". \n" + + " A special value of \"*\" means all users are allowed.</description>\n" + + " </property>\n" + + " <property>\n" + + " <name>emptyVal</name>\n" + + " <value></value>\n" + + " <description>non-empty-desc</description>\n" + + " </property>\n" + + " <property>\n" + + " <name>emptyDesc</name>\n" + + " <value></value>\n" + + " <description></description>\n" + + " </property>\n" + + " <property>\n" + + " <name>noDesc</name>\n" + + " <value></value>\n" + + " </property>\n" + + "</configuration>"; + + @Test + public void testParse() throws IOException { + + InputStream config_1 = new ByteArrayInputStream(config_1_str.getBytes()); + DefaultConfig config = new DefaultConfigParser().parse(config_1); + Assert.assertNotNull(config); + Assert.assertNotNull(config.getPropertyInfos()); + Assert.assertEquals(6, config.getPropertyInfos().size()); + for (PropertyInfo pInfo : config.getPropertyInfos()) { + if (pInfo.getName().equals("security.client.protocol.acl")) { + Assert.assertEquals("*", pInfo.getValue()); + Assert.assertTrue(pInfo.getDescription().startsWith("ACL for HRegionInterface ")); + } + if (pInfo.getName().equals("emptyVal")) { + Assert.assertEquals("", pInfo.getValue()); + Assert.assertEquals("non-empty-desc", pInfo.getDescription()); + } + if (pInfo.getName().equals("emptyDesc")) { + Assert.assertEquals("", pInfo.getValue()); + Assert.assertEquals("", pInfo.getDescription()); + } + if (pInfo.getName().equals("noDesc")) { + Assert.assertEquals("", pInfo.getValue()); + Assert.assertNull(pInfo.getDescription()); + } + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4beaeb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java new file mode 100644 index 0000000..ba1912a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent.application.metadata; + +import org.apache.slider.providers.agent.AgentProviderService; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import static org.mockito.Mockito.doReturn; + +/** + * + */ +public class TestMetainfoParser { + protected static final Logger log = + LoggerFactory.getLogger(TestMetainfoParser.class); + public static final String METAINFO_XML = + "/org/apache/slider/providers/agent/application/metadata/metainfo.xml"; + + @Test + public void testParse() throws IOException { + + InputStream resStream = this.getClass().getResourceAsStream( + METAINFO_XML); + MetainfoParser parser = new MetainfoParser(); + Metainfo metainfo = parser.fromXmlStream(resStream); + Assert.assertNotNull(metainfo); + Assert.assertNotNull(metainfo.getApplication()); + Application application = metainfo.getApplication(); + assert "STORM".equals(application.getName()); + assert 6 == application.getComponents().size(); + OSPackage pkg = application.getOSSpecifics().get(0).getPackages().get(0); + assert "tarball".equals(pkg.getType()); + assert "files/apache-storm-0.9.1.2.1.1.0-237.tar.gz".equals(pkg.getName()); + boolean found = false; + for (Component comp : application.getComponents()) { + if (comp != null && comp.getName().equals("NIMBUS")) { + found = true; + Assert.assertEquals(0, comp.getComponentExports().size()); + } + if (comp != null && comp.getName().equals("SUPERVISOR")) { + Assert.assertEquals(1, comp.getComponentExports().size()); + } + if (comp != null && comp.getName().equals("ANOTHER_COMPONENT")) { + assert 2 == comp.getCommands().size(); + assert "start command".equals(comp.getCommands().get(0).getExec()); + assert "START".equals(comp.getCommands().get(0).getName()); + assert "stop command".equals(comp.getCommands().get(1).getExec()); + assert "STOP".equals(comp.getCommands().get(1).getName()); + } + } + assert found; + Assert.assertEquals(0, application.getConfigFiles().size()); + assert 1 == application.getPackages().size(); + Package p = application.getPackages().get(0); + assert "tarball".equals(p.getType()); + assert "test-tarball-name.tgz".equals(p.getName()); + } + + @Test + public void testJsonParse() throws IOException { + String metaInfo1_json = "{\n" + + "\"schemaVersion\":\"2.2\",\n" + + "\"application\":{\n" + + "\"name\": \"MEMCACHED\"," + + "\"exportGroups\": [" + + "{" + + "\"name\": \"Servers\"," + + "\"exports\": [" + + "{" + + "\"name\": \"host_port\"," + + "\"value\": \"${MEMCACHED_HOST}:${site.global.port}\"" + + "}" + + "]" + + "}" + + "]," + + "\"components\": [" + + "{" + + "\"name\": \"MEMCACHED\"," + + "\"compExports\": \"Servers-host_port\"," + + "\"commands\": [" + + "{" + + "\"exec\": \"java -classpath /usr/myapps/memcached/*:/usr/lib/hadoop/lib/* com.thimbleware.jmemcached.Main\"" + + "}" + + "]" + + "}," + + "{" + + "\"name\": \"MEMCACHED2\"," + + "\"commands\": [" + + "{" + + "\"exec\": \"scripts/config.py\"," + + "\"type\": \"PYTHON\"," + + "\"name\": \"CONFIGURE\"" + + "}" + + "]," + + "\"dockerContainers\": [" + + "{" + + "\"name\": \"redis\"," + + "\"image\": \"dockerhub/redis\"," + + "\"options\": \"-net=bridge\"," + + "\"mounts\": [" + + "{" + + "\"containerMount\": \"/tmp/conf\"," + + "\"hostMount\": \"{$conf:@//site/global/app_root}/conf\"" + + "}" + + "]" + + "}" + + "]" + + "}" + + "]" + + "}" + + "}"; + + MetainfoParser parser = new MetainfoParser(); + Metainfo mInfo = parser.fromJsonString(metaInfo1_json); + Assert.assertEquals("2.2", mInfo.getSchemaVersion()); + + Application app = mInfo.getApplication(); + Assert.assertNotNull(app); + + Assert.assertEquals("MEMCACHED", app.getName()); + List<ExportGroup> egs = app.getExportGroups(); + Assert.assertEquals(1, egs.size()); + ExportGroup eg = egs.get(0); + Assert.assertEquals("Servers", eg.getName()); + List<Export> exports = eg.getExports(); + Assert.assertEquals(1, exports.size()); + Export export = exports.get(0); + Assert.assertEquals("host_port", export.getName()); + Assert.assertEquals("${MEMCACHED_HOST}:${site.global.port}", export.getValue()); + + List<Component> components = app.getComponents(); + Assert.assertEquals(2, components.size()); + + Component c1 = mInfo.getApplicationComponent("MEMCACHED"); + Assert.assertNotNull(c1); + Assert.assertEquals("MEMCACHED", c1.getName()); + Assert.assertEquals("Servers-host_port", c1.getCompExports()); + Assert.assertEquals(1, c1.getCommands().size()); + ComponentCommand cmd = c1.getCommands().get(0); + Assert.assertEquals("START", cmd.getName()); + Assert.assertEquals("SHELL", cmd.getType()); + Assert.assertEquals("java -classpath /usr/myapps/memcached/*:/usr/lib/hadoop/lib/* com.thimbleware.jmemcached.Main", + cmd.getExec()); + + Component c2 = mInfo.getApplicationComponent("MEMCACHED2"); + Assert.assertNotNull(c2); + Assert.assertEquals("MEMCACHED2", c2.getName()); + Assert.assertEquals(1, c2.getCommands().size()); + cmd = c2.getCommands().get(0); + Assert.assertEquals("CONFIGURE", cmd.getName()); + Assert.assertEquals("PYTHON", cmd.getType()); + Assert.assertEquals("scripts/config.py", cmd.getExec()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4beaeb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java new file mode 100644 index 0000000..a1986cd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.server.appmaster; + +import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.apache.slider.common.SliderKeys; +import org.apache.slider.core.conf.MapOperations; +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +/** + * + */ +public class TestServiceRecordAttributes extends Assert { + + @Test + public void testAppConfigProvidedServiceRecordAttributes() throws Exception { + Map<String, String> options = new HashMap<>(); + options.put("slider.some.arbitrary.option", "arbitrary value"); + options.put("service.record.attribute.one_attribute", "one_attribute_value"); + options.put("service.record.attribute.second_attribute", "second_attribute_value"); + MapOperations serviceProps = new MapOperations(SliderKeys.COMPONENT_AM, options); + options = new HashMap<>(); + options.put("some.component.attribute", "component_attribute_value"); + options.put("service.record.attribute.component_attribute", "component_attribute_value"); + MapOperations compProps = new MapOperations("TEST_COMP", options); + + SliderAppMaster appMaster = new SliderAppMaster(); + + ServiceRecord appServiceRecord = new ServiceRecord(); + + appMaster.setProvidedServiceRecordAttributes(serviceProps, appServiceRecord); + + assertNull("property should not be attribute", + appServiceRecord.get("slider.some.arbitrary.option")); + assertEquals("wrong value", "one_attribute_value", + appServiceRecord.get("one_attribute")); + assertEquals("wrong value", "second_attribute_value", + appServiceRecord.get("second_attribute")); + + ServiceRecord compServiceRecord = new ServiceRecord(); + + appMaster.setProvidedServiceRecordAttributes(compProps, compServiceRecord); + + assertNull("should not be attribute", + compServiceRecord.get("some.component.attribute")); + assertEquals("wrong value", "component_attribute_value", + compServiceRecord.get("component_attribute")); + + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
