Repository: hadoop Updated Branches: refs/heads/yarn-native-services 34aeea7df -> 60d02d388
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60d02d38/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/ReportingLoop.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/ReportingLoop.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/ReportingLoop.java deleted file mode 100644 index 096838d..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/ReportingLoop.java +++ /dev/null @@ -1,265 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.servicemonitor; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.io.IOException; -import java.util.List; - -/** - * This is the monitor service - */ -public final class ReportingLoop implements Runnable, ProbeReportHandler, MonitorKeys, Closeable { - protected static final Logger log = LoggerFactory.getLogger(ReportingLoop.class); - private final ProbeWorker worker; - private final Thread workerThread; - private final int reportInterval; - private final int probeTimeout; - private final int bootstrapTimeout; - private ProbeReportHandler reporter; - private final String name; - private volatile boolean mustExit; - - public ReportingLoop(String name, - ProbeReportHandler reporter, - List<Probe> probes, - List<Probe> dependencyProbes, - int probeInterval, - int reportInterval, - int probeTimeout, - int bootstrapTimeout) throws IOException { - this(name, - reporter, - new ProbeWorker(probes, dependencyProbes, probeInterval, bootstrapTimeout), - reportInterval, - probeTimeout); - } - - /** - * Create a new reporting loop -and bond the worker's ProbeReportHandler - * to us - * @param name - * @param reporter - * @param worker - * @param reportInterval - * @param probeTimeout - */ - public ReportingLoop(String name, - ProbeReportHandler reporter, - ProbeWorker worker, - int reportInterval, - int probeTimeout) throws IOException { - this.name = name; - this.reporter = reporter; - this.reportInterval = reportInterval; - this.probeTimeout = probeTimeout; - this.worker = worker; - this.bootstrapTimeout = worker.getBootstrapTimeout(); - worker.setReportHandler(this); - workerThread = new Thread(worker, "probe thread - " + name); - worker.init(); - } - - public int getBootstrapTimeout() { - return bootstrapTimeout; - } - - public ReportingLoop withReporter(ProbeReportHandler reporter) { - assert this.reporter == null : "attempting to reassign reporter "; - assert reporter != null : "new reporter is null"; - this.reporter = reporter; - return this; - } - - /** - * Start the monitoring. - * - * @return false if the monitoring did not start and that the worker threads - * should be run up. - */ - public boolean startReporting() { - String description = "Service Monitor for " + name + ", probe-interval= " - + MonitorUtils.millisToHumanTime(worker.interval) - + ", report-interval=" + MonitorUtils.millisToHumanTime(reportInterval) - + ", probe-timeout=" + timeoutToStr(probeTimeout) - + ", bootstrap-timeout=" + timeoutToStr(bootstrapTimeout); - log.info("Starting reporting" - + " to " + reporter - + description); - return reporter.commence(name, description); - } - - private String timeoutToStr(int timeout) { - return timeout >= 0 ? MonitorUtils.millisToHumanTime(timeout) : "not set"; - } - - private void startWorker() { - log.info("Starting reporting worker thread "); - workerThread.setDaemon(true); - workerThread.start(); - } - - - /** - * This exits the process cleanly - */ - @Override - public void close() { - log.info("Stopping reporting"); - mustExit = true; - if (worker != null) { - worker.setMustExit(); - workerThread.interrupt(); - } - if (reporter != null) { - reporter.unregister(); - } - } - - @Override - public void probeFailure(ProbeFailedException exception) { - reporter.probeFailure(exception); - } - - @Override - public void probeProcessStateChange(ProbePhase probePhase) { - reporter.probeProcessStateChange(probePhase); - } - - @Override - public void probeBooted(ProbeStatus status) { - reporter.probeBooted(status); - } - - private long now() { - return System.currentTimeMillis(); - } - - @Override - public void probeResult(ProbePhase phase, ProbeStatus status) { - reporter.probeResult(phase, status); - } - - @Override - public boolean commence(String n, String description) { - return true; - } - - @Override - public void unregister() { - } - - @Override - public void heartbeat(ProbeStatus status) { - } - - @Override - public void probeTimedOut(ProbePhase currentPhase, Probe probe, ProbeStatus lastStatus, - long currentTime) { - } - - @Override - public void liveProbeCycleCompleted() { - //delegate to the reporter - reporter.liveProbeCycleCompleted(); - } - - /** - * The reporting loop - */ - void reportingLoop() { - - while (!mustExit) { - try { - ProbeStatus workerStatus = worker.getLastStatus(); - long now = now(); - long lastStatusIssued = workerStatus.getTimestamp(); - long timeSinceLastStatusIssued = now - lastStatusIssued; - //two actions can occur here: a heartbeat is issued or a timeout reported. - //this flag decides which - boolean heartbeat; - - //based on phase, decide whether to heartbeat or timeout - ProbePhase probePhase = worker.getProbePhase(); - switch (probePhase) { - case DEPENDENCY_CHECKING: - //no timeouts in dependency phase - heartbeat = true; - break; - - case BOOTSTRAPPING: - //the timeout here is fairly straightforward: heartbeats are - //raised while the worker hasn't timed out - heartbeat = bootstrapTimeout < 0 || timeSinceLastStatusIssued < bootstrapTimeout; - - break; - - case LIVE: - //use the probe timeout interval between the current time - //and the time the last status event was received. - heartbeat = timeSinceLastStatusIssued < probeTimeout; - break; - - case INIT: - case TERMINATING: - default: - //send a heartbeat, because this isn't the time to be failing - heartbeat = true; - } - if (heartbeat) { - //a heartbeat is sent to the reporter - reporter.heartbeat(workerStatus); - } else { - //no response from the worker -it is hung. - reporter.probeTimedOut(probePhase, - worker.getCurrentProbe(), - workerStatus, - now - ); - } - - //now sleep - Thread.sleep(reportInterval); - - } catch (InterruptedException e) { - //interrupted -always exit the loop. - break; - } - } - //this point is reached if and only if a clean exit was requested or something failed. - } - - /** - * This can be run in a separate thread, or it can be run directly from the caller. - * Test runs do the latter, HAM runs multiple reporting threads. - */ - @Override - public void run() { - try { - startWorker(); - reportingLoop(); - } catch (RuntimeException e) { - log.warn("Failure in the reporting loop: " + e, e); - //rethrow so that inline code can pick it up (e.g. test runs) - throw e; - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/60d02d38/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java index 676db82..3033537 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java @@ -62,6 +62,10 @@ public interface RestApiErrorMessages { "Invalid no of containers specified"; String ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID = ERROR_CONTAINERS_COUNT_INVALID + ERROR_SUFFIX_FOR_COMPONENT; + String ERROR_DEPENDENCY_INVALID = "Dependency %s for component %s is " + + "invalid, does not exist as a component"; + String ERROR_DEPENDENCY_CYCLE = "Invalid dependencies, a cycle may " + + "exist: %s"; String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED = "Cannot specify" + " cpus/memory along with profile"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/60d02d38/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java index 80a31c0..e977727 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java @@ -32,14 +32,18 @@ import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.persist.JsonSerDeser; import org.apache.slider.providers.AbstractClientProvider; import org.apache.slider.providers.SliderProviderFactory; +import org.apache.slider.server.servicemonitor.MonitorUtils; import org.codehaus.jackson.map.PropertyNamingStrategy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Set; public class ServiceApiUtil { @@ -176,9 +180,22 @@ public class ServiceApiUtil { if (comp.getLaunchCommand() == null) { comp.setLaunchCommand(globalLaunchCommand); } + // validate dependency existence + if (comp.getDependencies() != null) { + for (String dependency : comp.getDependencies()) { + if (!componentNames.contains(dependency)) { + throw new IllegalArgumentException(String.format( + RestApiErrorMessages.ERROR_DEPENDENCY_INVALID, dependency, + comp.getName())); + } + } + } validateComponent(comp, fs.getFileSystem()); } + // validate dependency tree + sortByDependencies(application.getComponents()); + // Application lifetime if not specified, is set to unlimited lifetime if (application.getLifetime() == null) { application.setLifetime(RestApiConstants.DEFAULT_UNLIMITED_LIFETIME); @@ -207,6 +224,8 @@ public class ServiceApiUtil { } compClientProvider.validateConfigFiles(comp.getConfiguration() .getFiles(), fs); + + MonitorUtils.getProbe(comp.getReadinessCheck()); } @VisibleForTesting @@ -301,6 +320,67 @@ public class ServiceApiUtil { return comp; } + public static Collection<Component> sortByDependencies(List<Component> + components) { + Map<String, Component> sortedComponents = + sortByDependencies(components, null); + return sortedComponents.values(); + } + + /** + * Each internal call of sortByDependencies will identify all of the + * components with the same dependency depth (the lowest depth that has not + * been processed yet) and add them to the sortedComponents list, preserving + * their original ordering in the components list. + * + * So the first time it is called, all components with no dependencies + * (depth 0) will be identified. The next time it is called, all components + * that have dependencies only on the the depth 0 components will be + * identified (depth 1). This will be repeated until all components have + * been added to the sortedComponents list. If no new components are + * identified but the sortedComponents list is not complete, an error is + * thrown. + */ + private static Map<String, Component> sortByDependencies(List<Component> + components, Map<String, Component> sortedComponents) { + if (sortedComponents == null) { + sortedComponents = new LinkedHashMap<>(); + } + + Map<String, Component> componentsToAdd = new LinkedHashMap<>(); + List<Component> componentsSkipped = new ArrayList<>(); + for (Component component : components) { + String name = component.getName(); + if (sortedComponents.containsKey(name)) { + continue; + } + boolean dependenciesAlreadySorted = true; + if (!SliderUtils.isEmpty(component.getDependencies())) { + for (String dependency : component.getDependencies()) { + if (!sortedComponents.containsKey(dependency)) { + dependenciesAlreadySorted = false; + break; + } + } + } + if (dependenciesAlreadySorted) { + componentsToAdd.put(name, component); + } else { + componentsSkipped.add(component); + } + } + + if (componentsToAdd.size() == 0) { + throw new IllegalArgumentException(String.format(RestApiErrorMessages + .ERROR_DEPENDENCY_CYCLE, componentsSkipped)); + } + sortedComponents.putAll(componentsToAdd); + if (sortedComponents.size() == components.size()) { + return sortedComponents; + } + return sortByDependencies(components, sortedComponents); + } + public static String $(String s) { return "${" + s +"}"; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/60d02d38/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDependencies.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDependencies.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDependencies.java new file mode 100644 index 0000000..2967309 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDependencies.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.server.appmaster.model.appstate; + +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.slider.api.types.ApplicationLivenessInformation; +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; +import org.apache.slider.server.appmaster.model.mock.MockRoles; +import org.apache.slider.server.appmaster.operations.AbstractRMOperation; +import org.apache.slider.server.appmaster.state.ContainerAssignment; +import org.apache.slider.server.appmaster.state.RoleInstance; +import org.apache.slider.server.appmaster.state.RoleStatus; +import org.apache.slider.server.servicemonitor.ProbeStatus; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Test for postponing container requests until dependencies are ready. + */ +public class TestMockAppStateDependencies extends BaseMockAppStateTest + implements MockRoles { + + private org.apache.slider.server.servicemonitor.Probe successProbe = + new org.apache.slider.server.servicemonitor.Probe("success", null) { + @Override + public ProbeStatus ping(RoleInstance roleInstance) { + ProbeStatus status = new ProbeStatus(); + status.succeed(this); + return status; + } + }; + + private org.apache.slider.server.servicemonitor.Probe failureProbe = + new org.apache.slider.server.servicemonitor.Probe("failure", null) { + @Override + public ProbeStatus ping(RoleInstance roleInstance) { + ProbeStatus status = new ProbeStatus(); + status.fail(this, new Exception()); + return status; + } + }; + + @Override + public String getTestName() { + return "TestMockAppStateDependencies"; + } + + @Test + public void testDependencies() throws Throwable { + RoleStatus role0Status = getRole0Status(); + RoleStatus role1Status = getRole1Status(); + + // set desired instances for role0 to 1 + role0Status.setDesired(1); + // set probe for role0 to use a ping that will always succeed + role0Status.getProviderRole().probe = successProbe; + + // set desired instances for role1 to 1 + role1Status.setDesired(1); + // set role0 as a dependency of role1 + role1Status.getProviderRole().component.setDependencies(Collections + .singletonList(ROLE0)); + + // role0 has no dependencies, so its dependencies are ready + assertTrue(appState.areDependenciesReady(role0Status)); + // role1 dependency (role0) is not ready yet + assertFalse(appState.areDependenciesReady(role1Status)); + // start the single requested instance for role0 + review(ROLE0, 2); + + // role0 is still not ready because a ping has not been issued + assertFalse(appState.areDependenciesReady(role1Status)); + // issue pings + appState.monitorComponentInstances(); + // now role0 is ready + assertTrue(appState.areDependenciesReady(role1Status)); + // increase the desired containers for role0 + role0Status.setDesired(2); + // role0 is no longer ready + assertFalse(appState.areDependenciesReady(role1Status)); + // start a second instance for role0 + review(ROLE0, 2); + + // role0 is not ready because ping has not been issued for the new instance + assertFalse(appState.areDependenciesReady(role1Status)); + // issue pings + appState.monitorComponentInstances(); + // role0 is ready + assertTrue(appState.areDependenciesReady(role1Status)); + + // set probe for role0 to use a ping that will always fail + role0Status.getProviderRole().probe = failureProbe; + // issue pings + appState.monitorComponentInstances(); + // role0 is not ready (failure probe works) + assertFalse(appState.areDependenciesReady(role1Status)); + // set probe for role0 to use a ping that will always succeed + role0Status.getProviderRole().probe = successProbe; + // issue pings + appState.monitorComponentInstances(); + // role0 is ready + assertTrue(appState.areDependenciesReady(role1Status)); + + // now role1 instances can be started + review(ROLE1, 1); + } + + public void review(String expectedRole, int outstanding) throws Exception { + List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); + + // expect one request in the list + assertEquals(1, ops.size()); + // and in a liveness check, expected outstanding + ApplicationLivenessInformation liveness = + appState.getApplicationLivenessInformation(); + assertEquals(outstanding, liveness.requestsOutstanding); + assertFalse(liveness.allRequestsSatisfied); + + // record container allocated and verify it has the expected role + List<Container> allocations = engine.execute(ops); + List<ContainerAssignment> assignments = new ArrayList<>(); + List<AbstractRMOperation> releases = new ArrayList<>(); + appState.onContainersAllocated(allocations, assignments, releases); + assertEquals(1, assignments.size()); + ContainerAssignment assigned = assignments.get(0); + Container target = assigned.container; + RoleInstance ri = roleInstance(assigned); + assertEquals(expectedRole, ri.role); + + // one fewer request outstanding + liveness = appState.getApplicationLivenessInformation(); + assertEquals(outstanding - 1, liveness.requestsOutstanding); + + // record container start submitted + appState.containerStartSubmitted(target, ri); + + // additional review results in no additional requests + ops = appState.reviewRequestAndReleaseNodes(); + assertTrue(ops.isEmpty()); + + // record container start + appState.innerOnNodeManagerContainerStarted(target.getId()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/60d02d38/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java index 703d65f..edc1866 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java @@ -103,8 +103,6 @@ public class TestMockAppStateUniqueNames extends BaseMockAppStateTest assertEquals(i, instance.componentId); assertEquals(group, instance.role); assertEquals(group, instance.providerRole.name); - assertEquals(group, instance.providerRole.group); - // TODO remove group from provider role if it continues to be unused i++; } } @@ -124,7 +122,6 @@ public class TestMockAppStateUniqueNames extends BaseMockAppStateTest assertEquals(0, roleStatus.getDesired()); assertEquals(1024L, roleStatus.getResourceRequirements().getMemorySize()); assertEquals(2, roleStatus.getResourceRequirements().getVirtualCores()); - assertEquals("group1", roleStatus.getGroup()); // now flex back up appState.updateComponents(Collections.singletonMap("group1", 3L)); @@ -147,7 +144,6 @@ public class TestMockAppStateUniqueNames extends BaseMockAppStateTest RoleStatus group1 = appState.lookupRoleStatus("group1"); assertEquals(3, group1.getDesired()); assertEquals(1024L, group1.getResourceRequirements().getMemorySize()); - assertEquals("group1", group1.getGroup()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/60d02d38/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java index 7d8f5a7..555db75 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java @@ -343,7 +343,7 @@ public class TestRoleHistoryOutstandingRequestTracker extends public void testBuildResourceRequirements() throws Throwable { // Store original values Application application = appState.getClusterStatus(); - Component role0 = application.getComponent(getRole0Status().getGroup()); + Component role0 = application.getComponent(getRole0Status().getName()); String origMem = role0.getResource().getMemory(); Integer origVcores = role0.getResource().getCpus(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/60d02d38/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/servicemonitor/TestPortProbe.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/servicemonitor/TestPortProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/servicemonitor/TestPortProbe.java index 458d1bc..dacfb0a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/servicemonitor/TestPortProbe.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/servicemonitor/TestPortProbe.java @@ -17,20 +17,25 @@ package org.apache.slider.server.servicemonitor; -import org.apache.hadoop.conf.Configuration; +import org.apache.slider.server.appmaster.model.mock.MockFactory; +import org.apache.slider.server.appmaster.state.RoleInstance; import org.junit.Assert; import org.junit.Test; public class TestPortProbe extends Assert { + private final MockFactory factory = MockFactory.INSTANCE; + /** * Assert that a port probe failed if the port is closed * @throws Throwable */ @Test public void testPortProbeFailsClosedPort() throws Throwable { - PortProbe probe = new PortProbe("127.0.0.1", 65500, 100, "", new Configuration()); + PortProbe probe = new PortProbe(65500, 100); probe.init(); - ProbeStatus status = probe.ping(true); + RoleInstance roleInstance = new RoleInstance(factory.newContainer()); + roleInstance.ip = "127.0.0.1"; + ProbeStatus status = probe.ping(roleInstance); assertFalse("Expected a failure but got successful result: " + status, status.isSuccess()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/60d02d38/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java index 9ca3242..d7a9cfd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java @@ -34,6 +34,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; import static org.apache.slider.util.RestApiConstants.DEFAULT_COMPONENT_NAME; import static org.apache.slider.util.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME; @@ -390,4 +393,52 @@ public class TestServiceApiUtil { // original component replaced by external component assertNotNull(app.getComponent("comp1")); } + + public static void verifyDependencySorting(List<Component> components, + Component... expectedSorting) { + Collection<Component> actualSorting = ServiceApiUtil.sortByDependencies( + components); + assertEquals(expectedSorting.length, actualSorting.size()); + int i = 0; + for (Component component : actualSorting) { + assertEquals(expectedSorting[i++], component); + } + } + + @Test + public void testDependencySorting() throws IOException { + Component a = new Component().name("a"); + Component b = new Component().name("b"); + Component c = new Component().name("c"); + Component d = new Component().name("d").dependencies(Arrays.asList("c")); + Component e = new Component().name("e").dependencies(Arrays.asList("b", + "d")); + + verifyDependencySorting(Arrays.asList(a, b, c), a, b, c); + verifyDependencySorting(Arrays.asList(c, a, b), c, a, b); + verifyDependencySorting(Arrays.asList(a, b, c, d, e), a, b, c, d, e); + verifyDependencySorting(Arrays.asList(e, d, c, b, a), c, b, a, d, e); + + c.setDependencies(Arrays.asList("e")); + try { + verifyDependencySorting(Arrays.asList(a, b, c, d, e)); + Assert.fail(EXCEPTION_PREFIX + "components with dependency cycle"); + } catch (IllegalArgumentException ex) { + assertEquals(String.format( + RestApiErrorMessages.ERROR_DEPENDENCY_CYCLE, Arrays.asList(c, d, + e)), ex.getMessage()); + } + + SliderFileSystem sfs = initMock(null); + Application application = createValidApplication(null); + application.setComponents(Arrays.asList(c, d, e)); + try { + ServiceApiUtil.validateAndResolveApplication(application, sfs); + Assert.fail(EXCEPTION_PREFIX + "components with bad dependencies"); + } catch (IllegalArgumentException ex) { + assertEquals(String.format( + RestApiErrorMessages.ERROR_DEPENDENCY_INVALID, "b", "e"), ex + .getMessage()); + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
