Repository: incubator-slider Updated Branches: refs/heads/develop 39e04e36e -> bbd09a292
SLIDER-768: add new ApplicationLivenessInformation; do some cleanup & dedup of code. Note how AppState is directly using a Codahale counter for outstanding requests. Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/181c5e9f Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/181c5e9f Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/181c5e9f Branch: refs/heads/develop Commit: 181c5e9f8b67f7d13c444402ecd9f0a993065281 Parents: 922439e Author: Steve Loughran <ste...@apache.org> Authored: Fri Jan 30 14:23:02 2015 +0000 Committer: Steve Loughran <ste...@apache.org> Committed: Fri Jan 30 14:23:02 2015 +0000 ---------------------------------------------------------------------- .../types/ApplicationLivenessInformation.java | 38 ++++++ .../slider/api/types/ComponentInformation.java | 63 +++++++++ .../slider/api/types/ContainerInformation.java | 57 ++++++++ .../apache/slider/api/types/PingResource.java | 47 +++++++ .../types/SerializedComponentInformation.java | 41 ------ .../types/SerializedContainerInformation.java | 57 -------- .../client/rest/SliderApplicationAPI.java | 22 +-- .../appmaster/management/MeterAndCounter.java | 8 ++ .../slider/server/appmaster/state/AppState.java | 134 ++++++++++++++----- .../appmaster/state/ProviderAppState.java | 17 +++ .../server/appmaster/state/RoleInstance.java | 13 +- .../server/appmaster/state/RoleStatus.java | 23 +--- .../state/StateAccessForProviders.java | 38 ++++++ .../server/appmaster/web/rest/RestPaths.java | 3 + .../rest/application/ApplicationResource.java | 94 +++++++++++-- .../application/actions/RestActionPing.java | 3 +- .../application/actions/RestActionStop.java | 1 - .../resources/LiveComponentsRefresher.java | 22 +-- .../resources/LiveContainersRefresher.java | 10 +- .../resources/LiveStatisticsRefresher.java | 39 ++++++ .../application/resources/PingResource.java | 44 ------ .../agent/rest/JerseyTestDelegates.groovy | 29 ++-- .../slider/agent/rest/RestTestDelegates.groovy | 32 ++--- .../rest/SliderRestClientTestDelegates.groovy | 21 ++- .../TestMockAppStateAppRestIntegration.groovy | 6 +- 25 files changed, 570 insertions(+), 292 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java new file mode 100644 index 0000000..ab11263 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.api.types; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +/** + * Serialized information about liveness + * <p> + * If true liveness probes are implemented, this + * datatype can be extended to publish their details. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class ApplicationLivenessInformation { + public boolean allRequestsSatisfied; + public int requested; + +} + + http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java new file mode 100644 index 0000000..67a3053 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.api.types; + +import org.apache.slider.api.StatusKeys; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Serializable version of component data. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) + +public class ComponentInformation { + + public String name; + public int priority; + public int desired, actual, releasing; + public int requested; + public int failed, started, startFailed, completed, totalRequested; + public String failureMessage; + public int placementPolicy; + public List<String> containers; + + /** + * Build the statistics map from the current data + * @return a map for use in statistics reports + */ + public Map<String, Integer> buildStatistics() { + Map<String, Integer> stats = new HashMap<String, Integer>(); + stats.put(StatusKeys.STATISTICS_CONTAINERS_ACTIVE_REQUESTS, requested); + stats.put(StatusKeys.STATISTICS_CONTAINERS_COMPLETED, completed); + stats.put(StatusKeys.STATISTICS_CONTAINERS_DESIRED, desired); + stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED, failed); + stats.put(StatusKeys.STATISTICS_CONTAINERS_LIVE, actual); + stats.put(StatusKeys.STATISTICS_CONTAINERS_REQUESTED, totalRequested); + stats.put(StatusKeys.STATISTICS_CONTAINERS_STARTED, started); + stats.put(StatusKeys.STATISTICS_CONTAINERS_START_FAILED, startFailed); + return stats; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/main/java/org/apache/slider/api/types/ContainerInformation.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/types/ContainerInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/ContainerInformation.java new file mode 100644 index 0000000..14fa491 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/api/types/ContainerInformation.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.api.types; + +import org.apache.hadoop.registry.client.binding.JsonSerDeser; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + + +/** + * Serializable version of component instance data + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class ContainerInformation { + + public String containerId; + public String component; + public Boolean released; + public int state; + public Integer exitCode; + public String diagnostics; + public long createTime; + public long startTime; + + /** + * What is the tail output from the executed process (or [] if not started + * or the log cannot be picked up + */ + public String[] output; + public String host; + public String hostURL; + + @Override + public String toString() { + JsonSerDeser<ContainerInformation> serDeser = + new JsonSerDeser<ContainerInformation>( + ContainerInformation.class); + return serDeser.toString(this); + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/main/java/org/apache/slider/api/types/PingResource.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/types/PingResource.java b/slider-core/src/main/java/org/apache/slider/api/types/PingResource.java new file mode 100644 index 0000000..2a57ae0 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/api/types/PingResource.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.api.types; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +/** + * Serialized information to/from Ping operations + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class PingResource { + public long time; + public String text; + public String verb; + public String body; + + @Override + public String toString() { + + final StringBuilder sb = + new StringBuilder("PingResource{"); + sb.append("time=").append(time); + sb.append(", verb=").append(verb); + sb.append(", text='").append(text).append('\''); + sb.append(", body='").append(body).append('\''); + sb.append('}'); + return sb.toString(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/main/java/org/apache/slider/api/types/SerializedComponentInformation.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/types/SerializedComponentInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/SerializedComponentInformation.java deleted file mode 100644 index 8eac3db..0000000 --- a/slider-core/src/main/java/org/apache/slider/api/types/SerializedComponentInformation.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.api.types; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -import java.util.List; - -/** - * Serializable version of component data - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) - -public class SerializedComponentInformation { - - public String name; - public int priority; - public int desired, actual, requested, releasing; - public int failed, started, startFailed, completed, totalRequested; - public String failureMessage; - public int placementPolicy; - public List<String> containers; -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/main/java/org/apache/slider/api/types/SerializedContainerInformation.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/types/SerializedContainerInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/SerializedContainerInformation.java deleted file mode 100644 index e079e7a..0000000 --- a/slider-core/src/main/java/org/apache/slider/api/types/SerializedContainerInformation.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.api.types; - -import org.apache.hadoop.registry.client.binding.JsonSerDeser; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - - -/** - * Serializable version of role instance data - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -public class SerializedContainerInformation { - - public String containerId; - public String component; - public Boolean released; - public int state; - public Integer exitCode; - public String diagnostics; - public long createTime; - public long startTime; - - /** - * What is the tail output from the executed process (or [] if not started - * or the log cannot be picked up - */ - public String[] output; - public String host; - public String hostURL; - - @Override - public String toString() { - JsonSerDeser<SerializedContainerInformation> serDeser = - new JsonSerDeser<SerializedContainerInformation>( - SerializedContainerInformation.class); - return serDeser.toString(this); - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationAPI.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationAPI.java b/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationAPI.java index 16af359..514e5eb 100644 --- a/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationAPI.java +++ b/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationAPI.java @@ -24,13 +24,13 @@ import com.sun.jersey.api.client.GenericType; import com.sun.jersey.api.client.WebResource; import com.sun.jersey.api.representation.Form; import org.apache.commons.lang.StringUtils; -import org.apache.slider.api.types.SerializedComponentInformation; -import org.apache.slider.api.types.SerializedContainerInformation; +import org.apache.slider.api.types.ComponentInformation; +import org.apache.slider.api.types.ContainerInformation; import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTree; import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.core.restclient.HttpVerb; -import org.apache.slider.server.appmaster.web.rest.application.resources.PingResource; +import org.apache.slider.api.types.PingResource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -213,10 +213,10 @@ public class SliderApplicationAPI extends BaseRestClient { * @return a possibly empty list of serialized containers * @throws IOException on any failure */ - public Map<String, SerializedContainerInformation> enumContainers() throws + public Map<String, ContainerInformation> enumContainers() throws IOException { return getApplicationResource(LIVE_CONTAINERS, - new GenericType<Map<String, SerializedContainerInformation>>() { + new GenericType<Map<String, ContainerInformation>>() { }); } @@ -226,10 +226,10 @@ public class SliderApplicationAPI extends BaseRestClient { * @return the container information * @throws IOException on any failure */ - public SerializedContainerInformation getContainer( String containerId) throws + public ContainerInformation getContainer( String containerId) throws IOException { return getApplicationResource(LIVE_CONTAINERS + "/" + containerId, - SerializedContainerInformation.class); + ContainerInformation.class); } /** @@ -237,10 +237,10 @@ public class SliderApplicationAPI extends BaseRestClient { * @return a possibly empty map of components * @throws IOException on any failure */ - public Map<String, SerializedComponentInformation> enumComponents() throws + public Map<String, ComponentInformation> enumComponents() throws IOException { return getApplicationResource(LIVE_COMPONENTS, - new GenericType<Map<String, SerializedComponentInformation>>() { + new GenericType<Map<String, ComponentInformation>>() { }); } @@ -250,10 +250,10 @@ public class SliderApplicationAPI extends BaseRestClient { * @return the component details * @throws IOException on any failure */ - public SerializedComponentInformation getComponent(String componentName) throws + public ComponentInformation getComponent(String componentName) throws IOException { return getApplicationResource(LIVE_COMPONENTS + "/" + componentName, - SerializedComponentInformation.class); + ComponentInformation.class); } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MeterAndCounter.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MeterAndCounter.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MeterAndCounter.java index ed55f73..58d9eb3 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MeterAndCounter.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MeterAndCounter.java @@ -93,4 +93,12 @@ public class MeterAndCounter { public int hashCode() { return name.hashCode(); } + + /** + * Get the count. + * @return the current count + */ + public long getCount() { + return counter.getCount(); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index 95a7ca5..97a22b3 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -18,6 +18,7 @@ package org.apache.slider.server.appmaster.state; +import com.codahale.metrics.Counter; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; @@ -45,6 +46,8 @@ import org.apache.slider.api.ResourceKeys; import org.apache.slider.api.RoleKeys; import org.apache.slider.api.StatusKeys; import org.apache.slider.api.proto.Messages; +import org.apache.slider.api.types.ApplicationLivenessInformation; +import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.common.SliderExitCodes; import org.apache.slider.common.SliderKeys; import org.apache.slider.common.tools.ConfigHelper; @@ -158,12 +161,9 @@ public class AppState { private Map<String, String> clientProperties = new HashMap<String, String>(); /** - The cluster description published to callers - This is used as a synchronization point on activities that update - the CD, and also to update some of the structures that - feed in to the CD + * This is a template of the cluster status */ - private ClusterDescription clusterSpec = new ClusterDescription(); + private ClusterDescription clusterStatusTemplate = new ClusterDescription(); private final Map<Integer, RoleStatus> roleStatusMap = new ConcurrentHashMap<Integer, RoleStatus>(); @@ -222,6 +222,12 @@ public class AppState { /** + * Track the number of requested Containers + */ + private final Counter requestedContainerCount = new Counter(); + + + /** * Map of requested nodes. This records the command used to start it, * resources, etc. When container started callback is received, * the node is promoted from here to the containerMap @@ -278,7 +284,10 @@ public class AppState { private int failureThreshold = 10; private String logServerURL = ""; - + + /** + * Selector of containers to release; application wide. + */ private ContainerReleaseSelector containerReleaseSelector; /** @@ -372,20 +381,15 @@ public class AppState { } /** - * Get the desired cluster state - * @return the specification of the cluter + * Get the dynamcally created view of the cluster status + * @return */ - public ClusterDescription getClusterSpec() { - return clusterSpec; - } - - - public ClusterDescription getClusterStatus() { + public synchronized ClusterDescription getClusterStatus() { return clusterStatus; } @VisibleForTesting - protected void setClusterStatus(ClusterDescription clusterDesc) { + protected synchronized void setClusterStatus(ClusterDescription clusterDesc) { this.clusterStatus = clusterDesc; } @@ -458,18 +462,14 @@ public class AppState { return internalsSnapshot; } - public boolean isApplicationLive() { return applicationLive; } - - public long getSnapshotTime() { return snapshotTime; } - public AggregateConf getInstanceDefinitionSnapshot() { return instanceDefinitionSnapshot; } @@ -578,7 +578,7 @@ public class AppState { public void initClusterStatus() { //copy into cluster status. - ClusterDescription status = ClusterDescription.copy(clusterSpec); + ClusterDescription status = ClusterDescription.copy(clusterStatusTemplate); status.state = STATE_CREATED; MapOperations infoOps = new MapOperations("info", status.info); infoOps.mergeWithoutOverwrite(applicationInfo); @@ -677,14 +677,14 @@ public class AppState { internalsSnapshot.confTree); instanceDefinitionSnapshot.setName(instanceDefinition.getName()); - clusterSpec = + clusterStatusTemplate = ClusterDescriptionOperations.buildFromInstanceDefinition( instanceDefinition); // Add the -site configuration properties for (Map.Entry<String, String> prop : clientProperties.entrySet()) { - clusterSpec.clientProperties.put(prop.getKey(), prop.getValue()); + clusterStatusTemplate.clientProperties.put(prop.getKey(), prop.getValue()); } } @@ -1165,7 +1165,7 @@ public class AppState { * @param capability a resource to set up * @return the request for a new container */ - public AMRMClient.ContainerRequest buildContainerResourceAndRequest( + private AMRMClient.ContainerRequest buildContainerResourceAndRequest( RoleStatus role, Resource capability) { buildResourceRequirements(role, capability); @@ -1185,18 +1185,40 @@ public class AppState { * @param labelExpression label expression to satisfy * @return the container request to submit */ - public AMRMClient.ContainerRequest createContainerRequest(RoleStatus role, + private AMRMClient.ContainerRequest createContainerRequest(RoleStatus role, Resource resource, String labelExpression) { AMRMClient.ContainerRequest request; request = roleHistory.requestNode(role, resource, labelExpression); - role.incRequested(); + incrementRequestCount(role); return request; } + /** + * Increment the request count of a role. + * <p> + * Also updates application state counters + * @param role role being requested. + */ + protected void incrementRequestCount(RoleStatus role) { + role.incRequested(); + requestedContainerCount.inc(); + } + + /** + * dec requested count of a role + * <p> + * Also updates application state counters + * @param role role to decrement + */ + protected void decrementRequestCount(RoleStatus role) { + role.decRequested(); + requestedContainerCount.dec(); + } + /** * Get the value of a YARN requirement (cores, RAM, etc). @@ -1558,6 +1580,8 @@ public class AppState { } + + /** * Return the percentage done that Slider is to have YARN display in its * Web UI @@ -1630,12 +1654,36 @@ public class AppState { cd.statistics.put(rolename, stats); } + Map<String, Integer> sliderstats = getLiveStatistics(); + cd.statistics.put(SliderKeys.COMPONENT_AM, sliderstats); + return cd; + } + + /** + * get application liveness information + * @return a snapshot of the current liveness information + */ + public ApplicationLivenessInformation getApplicationLivenessInformation() { + ApplicationLivenessInformation li = new ApplicationLivenessInformation(); + int outstanding = (int) requestedContainerCount.getCount(); + li.requested = outstanding; + li.allRequestsSatisfied = outstanding == 0; + return li; + } + + /** + * Get the live statistics map + * @return a map of statistics values, defined in the {@link StatusKeys} + * keylist. + */ + protected Map<String, Integer> getLiveStatistics() { Map<String, Integer> sliderstats = new HashMap<String, Integer>(); sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_COMPLETED, completedContainerCount.get()); sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED, failedContainerCount.get()); - sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_LIVE, liveNodes.size()); + sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_LIVE, + liveNodes.size()); sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_STARTED, startedContainers.get()); sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_START_FAILED, @@ -1644,8 +1692,29 @@ public class AppState { surplusContainers.get()); sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_UNKNOWN_COMPLETED, completionOfUnknownContainerEvent.get()); - cd.statistics.put(SliderKeys.COMPONENT_AM, sliderstats); - return cd; + return sliderstats; + } + + /** + * Get a snapshot of component information. + * <p> + * This does <i>not</i> include any container list, which + * is more expensive to create. + * @return a map of current role status values. + */ + public Map<String, ComponentInformation> getComponentInfoSnapshot() { + + Map<Integer, RoleStatus> statusMap = getRoleStatusMap(); + Map<String, ComponentInformation> results = + new HashMap<String, ComponentInformation>( + statusMap.size()); + + for (RoleStatus status : statusMap.values()) { + String name = status.getName(); + ComponentInformation info = status.serialize(); + results.put(name, info); + } + return results; } /** @@ -1720,8 +1789,11 @@ public class AppState { /** * Look at the allocation status of one role, and trigger add/release - * actions if the number of desired role instances doesnt equal - * (actual+pending) + * actions if the number of desired role instances doesn't equal + * (actual+pending). + * <p> + * MUST be executed from within a synchronized method + * <p> * @param role role * @return a list of operations * @throws SliderInternalStateException if the operation reveals that @@ -1952,7 +2024,7 @@ public class AppState { //dec requested count - role.decRequested(); + decrementRequestCount(role); //inc allocated count -this may need to be dropped in a moment, // but us needed to update the logic below allocated = role.incActual(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java index 43ff52f..552e0a4 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java @@ -22,6 +22,8 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.slider.api.ClusterDescription; +import org.apache.slider.api.types.ApplicationLivenessInformation; +import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.core.exceptions.NoSuchNodeException; @@ -220,4 +222,19 @@ public class ProviderAppState implements StateAccessForProviders { public List<RoleStatus> cloneRoleStatusList() { return appState.cloneRoleStatusList(); } + + @Override + public ApplicationLivenessInformation getApplicationLivenessInformation() { + return appState.getApplicationLivenessInformation(); + } + + @Override + public Map<String, Integer> getLiveStatistics() { + return appState.getLiveStatistics(); + } + + @Override + public Map<String, ComponentInformation> getComponentInfoSnapshot() { + return appState.getComponentInfoSnapshot(); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java index ad4af1d..74abe72 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java @@ -25,9 +25,8 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; import org.apache.hadoop.registry.client.types.Endpoint; import org.apache.hadoop.registry.client.types.ProtocolTypes; -import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.proto.Messages; -import org.apache.slider.api.types.SerializedContainerInformation; +import org.apache.slider.api.types.ContainerInformation; import org.apache.slider.common.tools.SliderUtils; import java.util.ArrayList; @@ -56,10 +55,14 @@ public final class RoleInstance implements Cloneable { * Name of the role */ public String role; + + /** + * Role Id; matches priority in resources.json + */ public int roleId; /** - * state from {@link StateValues} + * state from StateValues */ public int state; @@ -239,8 +242,8 @@ public final class RoleInstance implements Cloneable { * may be shared * @return a serialized form for marshalling as JSON */ - public SerializedContainerInformation serialize() { - SerializedContainerInformation info = new SerializedContainerInformation(); + public ContainerInformation serialize() { + ContainerInformation info = new ContainerInformation(); info.containerId = id; info.component = role; info.startTime = startTime; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java index 3edc5f1..fcc3ae0 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java @@ -18,14 +18,12 @@ package org.apache.slider.server.appmaster.state; -import org.apache.slider.api.StatusKeys; -import org.apache.slider.api.types.SerializedComponentInformation; +import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.providers.PlacementPolicy; import org.apache.slider.providers.ProviderRole; import java.io.Serializable; import java.util.Comparator; -import java.util.HashMap; import java.util.Map; @@ -272,25 +270,17 @@ public final class RoleStatus implements Cloneable { * Build the statistics map from the current data * @return a map for use in statistics reports */ - public synchronized Map<String, Integer> buildStatistics() { - Map<String, Integer> stats = new HashMap<String, Integer>(); - stats.put(StatusKeys.STATISTICS_CONTAINERS_ACTIVE_REQUESTS, getRequested()); - stats.put(StatusKeys.STATISTICS_CONTAINERS_COMPLETED, getCompleted()); - stats.put(StatusKeys.STATISTICS_CONTAINERS_DESIRED, getDesired()); - stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED, getFailed()); - stats.put(StatusKeys.STATISTICS_CONTAINERS_LIVE, getActual()); - stats.put(StatusKeys.STATISTICS_CONTAINERS_REQUESTED, getTotalRequested()); - stats.put(StatusKeys.STATISTICS_CONTAINERS_STARTED, getStarted()); - stats.put(StatusKeys.STATISTICS_CONTAINERS_START_FAILED, getStartFailed()); - return stats; + public Map<String, Integer> buildStatistics() { + ComponentInformation componentInformation = serialize(); + return componentInformation.buildStatistics(); } /** * Produced a serialized form which can be served up as JSON * @return a summary of the current role status. */ - public synchronized SerializedComponentInformation serialize() { - SerializedComponentInformation info = new SerializedComponentInformation(); + public synchronized ComponentInformation serialize() { + ComponentInformation info = new ComponentInformation(); info.name = name; info.priority = getPriority(); info.desired = desired; @@ -299,7 +289,6 @@ public final class RoleStatus implements Cloneable { info.releasing = releasing; info.failed = failed; info.startFailed = startFailed; - info.requested = requested; info.placementPolicy = getPlacementPolicy(); info.failureMessage = failureMessage; info.totalRequested = totalRequested; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java index 057c07b..13c67b0 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java @@ -22,6 +22,9 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.slider.api.ClusterDescription; +import org.apache.slider.api.StatusKeys; +import org.apache.slider.api.types.ApplicationLivenessInformation; +import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.core.exceptions.NoSuchNodeException; @@ -37,6 +40,10 @@ import java.util.Map; */ public interface StateAccessForProviders { + /** + * Get a map of role status entries by role Id + * @return the map of currently defined roles. + */ Map<Integer, RoleStatus> getRoleStatusMap(); /** @@ -77,8 +84,17 @@ public interface StateAccessForProviders { */ List<String> listConfigSets(); + /** + * Get a map of all the failed nodes + * @return map of recorded failed notes + */ Map<ContainerId, RoleInstance> getFailedNodes(); + /** + * Get the live nodes. + * + * @return the live nodes + */ Map<ContainerId, RoleInstance> getLiveNodes(); /** @@ -220,4 +236,26 @@ public interface StateAccessForProviders { * @return a snapshot of the role status entries */ List<RoleStatus> cloneRoleStatusList(); + + /** + * get application liveness information + * @return a snapshot of the current liveness information + */ + ApplicationLivenessInformation getApplicationLivenessInformation(); + + /** + * Get the live statistics map + * @return a map of statistics values, defined in the {@link StatusKeys} + * keylist. + */ + Map<String, Integer> getLiveStatistics(); + + /** + * Get a snapshot of component information. + * <p> + * This does <i>not</i> include any container list, which + * is more expensive to create. + * @return a map of current role status values. + */ + Map<String, ComponentInformation> getComponentInfoSnapshot(); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java index 35f3e13..06b7ba2 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java @@ -134,6 +134,9 @@ public class RestPaths { public static final String LIVE_RESOURCES = "/live/resources"; public static final String LIVE_CONTAINERS = "/live/containers"; public static final String LIVE_COMPONENTS = "/live/components"; + public static final String LIVE_NODES = "/live/"; + public static final String LIVE_LIVENESS = "/live/liveness"; + public static final String LIVE_STATISTICS = "/live/statistics"; public static final String MODEL = "/model"; public static final String MODEL_DESIRED = MODEL +"/desired"; public static final String MODEL_DESIRED_APPCONF = MODEL_DESIRED +"/appconf"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java index 735fd48..6667548 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java @@ -21,8 +21,9 @@ package org.apache.slider.server.appmaster.web.rest.application; import com.google.common.collect.Lists; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.webapp.NotFoundException; -import org.apache.slider.api.types.SerializedComponentInformation; -import org.apache.slider.api.types.SerializedContainerInformation; +import org.apache.slider.api.types.ApplicationLivenessInformation; +import org.apache.slider.api.types.ComponentInformation; +import org.apache.slider.api.types.ContainerInformation; import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTree; import org.apache.slider.core.exceptions.NoSuchNodeException; @@ -43,7 +44,8 @@ import org.apache.slider.server.appmaster.web.rest.application.resources.Content import org.apache.slider.server.appmaster.web.rest.application.resources.LiveComponentsRefresher; import org.apache.slider.server.appmaster.web.rest.application.resources.LiveResourcesRefresher; import org.apache.slider.server.appmaster.web.rest.application.actions.RestActionPing; -import org.apache.slider.server.appmaster.web.rest.application.resources.PingResource; +import org.apache.slider.api.types.PingResource; +import org.apache.slider.server.appmaster.web.rest.application.resources.LiveStatisticsRefresher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,6 +60,7 @@ import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; +import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Context; import static javax.ws.rs.core.MediaType.*; @@ -72,7 +75,7 @@ public class ApplicationResource extends AbstractSliderResource { private static final Logger log = LoggerFactory.getLogger(ApplicationResource.class); - public static final int LIFESPAN = 1000; + public static final int LIFESPAN = 500; public static final List<String> LIVE_ENTRIES = toJsonList("resources", "containers", "components", @@ -102,10 +105,10 @@ public class ApplicationResource extends AbstractSliderResource { new CachedContent<ConfTree>(LIFESPAN, new LiveResourcesRefresher(state))); cache.put(LIVE_CONTAINERS, - new CachedContent<Map<String, SerializedContainerInformation>>(LIFESPAN, + new CachedContent<Map<String, ContainerInformation>>(LIFESPAN, new LiveContainersRefresher(state))); cache.put(LIVE_COMPONENTS, - new CachedContent<Map<String, SerializedComponentInformation>> (LIFESPAN, + new CachedContent<Map<String, ComponentInformation>> (LIFESPAN, new LiveComponentsRefresher(state))); cache.put(MODEL_DESIRED, new CachedContent<AggregateConf>(LIFESPAN, @@ -125,6 +128,9 @@ public class ApplicationResource extends AbstractSliderResource { cache.put(MODEL_DESIRED_RESOURCES, new CachedContent<ConfTree>(LIFESPAN, new AppconfRefresher(state, true, true))); + cache.put(LIVE_STATISTICS, + new CachedContent<Map<String, Integer>>(LIFESPAN, + new LiveStatisticsRefresher(state))); } /** @@ -223,10 +229,10 @@ public class ApplicationResource extends AbstractSliderResource { @GET @Path(LIVE_CONTAINERS) @Produces({APPLICATION_JSON}) - public Map<String, SerializedContainerInformation> getLiveContainers() { + public Map<String, ContainerInformation> getLiveContainers() { markGet(SLIDER_SUBPATH_APPLICATION, LIVE_CONTAINERS); try { - return (Map<String, SerializedContainerInformation>)cache.lookup( + return (Map<String, ContainerInformation>)cache.lookup( LIVE_CONTAINERS); } catch (Exception e) { throw buildException(LIVE_CONTAINERS, e); @@ -236,7 +242,7 @@ public class ApplicationResource extends AbstractSliderResource { @GET @Path(LIVE_CONTAINERS + "/{containerId}") @Produces({APPLICATION_JSON}) - public SerializedContainerInformation getLiveContainer( + public ContainerInformation getLiveContainer( @PathParam("containerId") String containerId) { markGet(SLIDER_SUBPATH_APPLICATION, LIVE_CONTAINERS); try { @@ -252,10 +258,10 @@ public class ApplicationResource extends AbstractSliderResource { @GET @Path(LIVE_COMPONENTS) @Produces({APPLICATION_JSON}) - public Map<String, SerializedComponentInformation> getLiveComponents() { + public Map<String, ComponentInformation> getLiveComponents() { markGet(SLIDER_SUBPATH_APPLICATION, LIVE_COMPONENTS); try { - return (Map<String, SerializedComponentInformation>) cache.lookup( + return (Map<String, ComponentInformation>) cache.lookup( LIVE_COMPONENTS); } catch (Exception e) { throw buildException(LIVE_COMPONENTS, e); @@ -265,12 +271,12 @@ public class ApplicationResource extends AbstractSliderResource { @GET @Path(LIVE_COMPONENTS + "/{component}") @Produces({APPLICATION_JSON}) - public SerializedComponentInformation getLiveComponent( + public ComponentInformation getLiveComponent( @PathParam("component") String component) { markGet(SLIDER_SUBPATH_APPLICATION, LIVE_COMPONENTS); try { RoleStatus roleStatus = state.lookupRoleStatus(component); - SerializedComponentInformation info = roleStatus.serialize(); + ComponentInformation info = roleStatus.serialize(); List<RoleInstance> containers = lookupRoleContainers(component); info.containers = new ArrayList<String>(containers.size()); for (RoleInstance container : containers) { @@ -280,10 +286,65 @@ public class ApplicationResource extends AbstractSliderResource { } catch (YarnRuntimeException e) { throw new NotFoundException("Unknown component: " + component); } catch (Exception e) { + throw buildException(LIVE_CONTAINERS +"/" + component, e); + } + } + + /** + * Liveness information for the application as a whole + * @return snapshot of liveness + */ + @GET + @Path(LIVE_LIVENESS) + @Produces({APPLICATION_JSON}) + public ApplicationLivenessInformation getLivenessInformation() { + markGet(SLIDER_SUBPATH_APPLICATION, LIVE_LIVENESS); + try { + return state.getApplicationLivenessInformation(); + } catch (Exception e) { throw buildException(LIVE_CONTAINERS, e); } } +/* +TODO: decide what structure to return here, then implement + + @GET + @Path(LIVE_LIVENESS + "/{component}") + @Produces({APPLICATION_JSON}) + public ApplicationLivenessInformation getLivenessForComponent( + @PathParam("component") String component) { + markGet(SLIDER_SUBPATH_APPLICATION, LIVE_COMPONENTS); + try { + RoleStatus roleStatus = state.lookupRoleStatus(component); + ApplicationLivenessInformation info = new ApplicationLivenessInformation(); + info.requested = roleStatus.getRequested(); + info.allRequestsSatisfied = info.requested == 0; + return info; + } catch (YarnRuntimeException e) { + throw new NotFoundException("Unknown component: " + component); + } catch (Exception e) { + throw buildException(LIVE_LIVENESS + "/" + component, e); + } + } +*/ + + /** + * Statistics of the application + * @return snapshot statistics + */ + @GET + @Path(LIVE_STATISTICS) + @Produces({APPLICATION_JSON}) + public Map<String, Integer> getLiveStatistics() { + markGet(SLIDER_SUBPATH_APPLICATION, LIVE_LIVENESS); + try { + return (Map<String, Integer>) cache.lookup(LIVE_STATISTICS); + } catch (Exception e) { + throw buildException(LIVE_STATISTICS, e); + } + } + /** * Look up all containers of a specific component name * @param component component/role name @@ -302,6 +363,13 @@ public class ApplicationResource extends AbstractSliderResource { return matching; } + /** + * Helper method; look up an aggregate configuration in the cache from + * a key, or raise an exception + * @param key key to resolve + * @return the configuration + * @throws WebApplicationException on a failure + */ protected AggregateConf lookupAggregateConf(String key) { try { return (AggregateConf) cache.lookup(key); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/RestActionPing.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/RestActionPing.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/RestActionPing.java index f20f296..80a0a1c 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/RestActionPing.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/RestActionPing.java @@ -18,12 +18,11 @@ package org.apache.slider.server.appmaster.web.rest.application.actions; -import org.apache.slider.server.appmaster.web.rest.application.resources.PingResource; +import org.apache.slider.api.types.PingResource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.core.Context; import javax.ws.rs.core.UriInfo; import java.util.Locale; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/RestActionStop.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/RestActionStop.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/RestActionStop.java index f94c983..c0e30cc 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/RestActionStop.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/RestActionStop.java @@ -22,7 +22,6 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.slider.core.main.LauncherExitCodes; import org.apache.slider.server.appmaster.actions.ActionStopSlider; import org.apache.slider.server.appmaster.web.WebAppApi; -import org.apache.slider.server.appmaster.web.rest.application.resources.PingResource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveComponentsRefresher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveComponentsRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveComponentsRefresher.java index 530bbd3..b6627a7 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveComponentsRefresher.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveComponentsRefresher.java @@ -18,15 +18,13 @@ package org.apache.slider.server.appmaster.web.rest.application.resources; -import org.apache.slider.api.types.SerializedComponentInformation; -import org.apache.slider.server.appmaster.state.RoleStatus; +import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.server.appmaster.state.StateAccessForProviders; -import java.util.HashMap; import java.util.Map; public class LiveComponentsRefresher - implements ResourceRefresher<Map<String, SerializedComponentInformation>> { + implements ResourceRefresher<Map<String, ComponentInformation>> { private final StateAccessForProviders state; @@ -35,19 +33,7 @@ public class LiveComponentsRefresher } @Override - public Map<String, SerializedComponentInformation> refresh() throws - Exception { - - Map<Integer, RoleStatus> roleStatusMap = state.getRoleStatusMap(); - Map<String, SerializedComponentInformation> results = - new HashMap<String, SerializedComponentInformation>( - roleStatusMap.size()); - - for (RoleStatus status : roleStatusMap.values()) { - String name = status.getName(); - SerializedComponentInformation info = status.serialize(); - results.put(name, info); - } - return results; + public Map<String, ComponentInformation> refresh() { + return state.getComponentInfoSnapshot(); } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java index 1b29801..276e8cc 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java @@ -18,7 +18,7 @@ package org.apache.slider.server.appmaster.web.rest.application.resources; -import org.apache.slider.api.types.SerializedContainerInformation; +import org.apache.slider.api.types.ContainerInformation; import org.apache.slider.server.appmaster.state.RoleInstance; import org.apache.slider.server.appmaster.state.StateAccessForProviders; @@ -29,7 +29,7 @@ import java.util.Map; /** * Refresh the container list. */ -public class LiveContainersRefresher implements ResourceRefresher<Map<String, SerializedContainerInformation>> { +public class LiveContainersRefresher implements ResourceRefresher<Map<String, ContainerInformation>> { private final StateAccessForProviders state; @@ -38,13 +38,13 @@ public class LiveContainersRefresher implements ResourceRefresher<Map<String, Se } @Override - public Map<String, SerializedContainerInformation> refresh() throws + public Map<String, ContainerInformation> refresh() throws Exception { List<RoleInstance> containerList = state.cloneOwnedContainerList(); - Map<String, SerializedContainerInformation> map = new HashMap<String, SerializedContainerInformation>(); + Map<String, ContainerInformation> map = new HashMap<String, ContainerInformation>(); for (RoleInstance instance : containerList) { - SerializedContainerInformation serialized = instance.serialize(); + ContainerInformation serialized = instance.serialize(); map.put(serialized.containerId, serialized); } return map; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveStatisticsRefresher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveStatisticsRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveStatisticsRefresher.java new file mode 100644 index 0000000..d31b455 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveStatisticsRefresher.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.web.rest.application.resources; + +import org.apache.slider.server.appmaster.state.StateAccessForProviders; + +import java.util.Map; + +public class LiveStatisticsRefresher implements ResourceRefresher<Map<String,Integer>> { + + private final StateAccessForProviders state; + + public LiveStatisticsRefresher(StateAccessForProviders state) { + this.state = state; + } + + @Override + public Map<String, Integer> refresh() throws Exception { + + // snapshot resources + return state.getLiveStatistics(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/PingResource.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/PingResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/PingResource.java deleted file mode 100644 index 3f67c55..0000000 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/PingResource.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.web.rest.application.resources; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -public class PingResource { - public long time; - public String text; - public String verb; - public String body; - - @Override - public String toString() { - - final StringBuilder sb = - new StringBuilder("PingResource{"); - sb.append("time=").append(time); - sb.append(", verb=").append(verb); - sb.append(", text='").append(text).append('\''); - sb.append(", body='").append(body).append('\''); - sb.append('}'); - return sb.toString(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy index 40f86a8..1ff7eae 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy @@ -23,20 +23,19 @@ import com.sun.jersey.api.client.Client import com.sun.jersey.api.client.ClientResponse import com.sun.jersey.api.client.UniformInterfaceException import com.sun.jersey.api.client.WebResource -import com.sun.jersey.client.impl.ClientRequestImpl import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.hadoop.yarn.webapp.NotFoundException import org.apache.slider.api.StateValues -import org.apache.slider.api.types.SerializedComponentInformation -import org.apache.slider.api.types.SerializedContainerInformation +import org.apache.slider.api.types.ComponentInformation +import org.apache.slider.api.types.ContainerInformation import org.apache.slider.common.tools.SliderUtils import org.apache.slider.core.conf.AggregateConf import org.apache.slider.core.conf.ConfTree import org.apache.slider.core.conf.ConfTreeOperations import org.apache.slider.core.restclient.HttpVerb import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource -import org.apache.slider.server.appmaster.web.rest.application.resources.PingResource +import org.apache.slider.api.types.PingResource import org.apache.slider.test.SliderTestUtils import javax.ws.rs.core.MediaType @@ -237,12 +236,12 @@ class JerseyTestDelegates extends SliderTestUtils { public void testLiveContainers() throws Throwable { describe "Application REST ${LIVE_CONTAINERS}" - Map<String, SerializedContainerInformation> containers = + Map<String, ContainerInformation> containers = jGetApplicationResource(LIVE_CONTAINERS, HashMap) assert containers.size() == 1 log.info "${containers}" - SerializedContainerInformation amContainerInfo = - (SerializedContainerInformation) containers.values()[0] + ContainerInformation amContainerInfo = + (ContainerInformation) containers.values()[0] assert amContainerInfo.containerId def amContainerId = amContainerInfo.containerId @@ -257,10 +256,10 @@ class JerseyTestDelegates extends SliderTestUtils { describe "containers" - SerializedContainerInformation retrievedContainerInfo = + ContainerInformation retrievedContainerInfo = jFetchType( LIVE_CONTAINERS + "/${amContainerId}", - SerializedContainerInformation + ContainerInformation ) assert retrievedContainerInfo.containerId == amContainerId @@ -268,7 +267,7 @@ class JerseyTestDelegates extends SliderTestUtils { try { def result = jFetchType( LIVE_CONTAINERS + "/unknown", - SerializedContainerInformation + ContainerInformation ) fail("expected an error, got $result") } catch (NotFoundException e) { @@ -278,18 +277,18 @@ class JerseyTestDelegates extends SliderTestUtils { describe "components" - Map<String, SerializedComponentInformation> components = + Map<String, ComponentInformation> components = jFetchType(LIVE_COMPONENTS, HashMap) // two components assert components.size() >= 1 log.info "${components}" - SerializedComponentInformation amComponentInfo = - (SerializedComponentInformation) components[COMPONENT_AM] + ComponentInformation amComponentInfo = + (ComponentInformation) components[COMPONENT_AM] - SerializedComponentInformation amFullInfo = jFetchType( + ComponentInformation amFullInfo = jFetchType( LIVE_COMPONENTS + "/${COMPONENT_AM}", - SerializedComponentInformation + ComponentInformation ) assert amFullInfo.containers.size() == 1 http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy index f90e1ad..be0b3d8 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy @@ -18,15 +18,12 @@ package org.apache.slider.agent.rest -import com.sun.jersey.api.client.ClientResponse -import com.sun.jersey.api.client.WebResource import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.hadoop.yarn.webapp.NotFoundException -import org.apache.http.entity.ContentType import org.apache.slider.api.StateValues -import org.apache.slider.api.types.SerializedComponentInformation -import org.apache.slider.api.types.SerializedContainerInformation +import org.apache.slider.api.types.ComponentInformation +import org.apache.slider.api.types.ContainerInformation import org.apache.slider.core.conf.AggregateConf import org.apache.slider.core.conf.ConfTree import org.apache.slider.core.conf.ConfTreeOperations @@ -35,12 +32,11 @@ import org.apache.slider.core.restclient.HttpVerb import org.apache.slider.core.restclient.UrlConnectionOperations import org.apache.slider.server.appmaster.web.rest.RestPaths import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource -import org.apache.slider.server.appmaster.web.rest.application.resources.PingResource +import org.apache.slider.api.types.PingResource import org.apache.slider.test.Outcome import org.apache.slider.test.SliderTestUtils import javax.ws.rs.core.MediaType -import java.nio.charset.Charset import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES import static org.apache.slider.api.StatusKeys.* @@ -113,12 +109,12 @@ class RestTestDelegates extends SliderTestUtils { public void testLiveContainers() throws Throwable { describe "Application REST ${LIVE_CONTAINERS}" - Map<String, SerializedContainerInformation> containers = + Map<String, ContainerInformation> containers = fetchType(HashMap, appmaster, LIVE_CONTAINERS) assert containers.size() == 1 log.info "${containers}" - SerializedContainerInformation amContainerInfo = - (SerializedContainerInformation) containers.values()[0] + ContainerInformation amContainerInfo = + (ContainerInformation) containers.values()[0] assert amContainerInfo.containerId def amContainerId = amContainerInfo.containerId @@ -133,14 +129,14 @@ class RestTestDelegates extends SliderTestUtils { describe "containers" - SerializedContainerInformation retrievedContainerInfo = - fetchType(SerializedContainerInformation, appmaster, + ContainerInformation retrievedContainerInfo = + fetchType(ContainerInformation, appmaster, LIVE_CONTAINERS + "/${amContainerId}") assert retrievedContainerInfo.containerId == amContainerId // fetch missing try { - def result = fetchType(SerializedContainerInformation, appmaster, + def result = fetchType(ContainerInformation, appmaster, LIVE_CONTAINERS + "/unknown") fail("expected an error, got $result") } catch (NotFoundException e) { @@ -150,17 +146,17 @@ class RestTestDelegates extends SliderTestUtils { describe "components" - Map<String, SerializedComponentInformation> components = + Map<String, ComponentInformation> components = fetchType(HashMap, appmaster, LIVE_COMPONENTS) // two components assert components.size() >= 1 log.info "${components}" - SerializedComponentInformation amComponentInfo = - (SerializedComponentInformation) components[COMPONENT_AM] + ComponentInformation amComponentInfo = + (ComponentInformation) components[COMPONENT_AM] - SerializedComponentInformation amFullInfo = fetchType( - SerializedComponentInformation, + ComponentInformation amFullInfo = fetchType( + ComponentInformation, appmaster, LIVE_COMPONENTS + "/${COMPONENT_AM}") http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy index ef070e0..95943f4 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy @@ -22,10 +22,9 @@ import com.sun.jersey.api.client.Client import com.sun.jersey.api.client.WebResource import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import org.apache.hadoop.fs.PathNotFoundException import org.apache.slider.api.StateValues -import org.apache.slider.api.types.SerializedComponentInformation -import org.apache.slider.api.types.SerializedContainerInformation +import org.apache.slider.api.types.ComponentInformation +import org.apache.slider.api.types.ContainerInformation import org.apache.slider.client.rest.SliderApplicationAPI import org.apache.slider.core.conf.ConfTree import org.apache.slider.core.conf.ConfTreeOperations @@ -102,11 +101,11 @@ class SliderRestClientTestDelegates extends SliderTestUtils { public void testLiveContainers() throws Throwable { describe "Application REST ${LIVE_CONTAINERS}" - Map<String, SerializedContainerInformation> containers = appAPI.enumContainers() + Map<String, ContainerInformation> containers = appAPI.enumContainers() assert containers.size() == 1 log.info "${containers}" - SerializedContainerInformation amContainerInfo = - (SerializedContainerInformation) containers.values()[0] + ContainerInformation amContainerInfo = + (ContainerInformation) containers.values()[0] assert amContainerInfo.containerId def amContainerId = amContainerInfo.containerId @@ -121,7 +120,7 @@ class SliderRestClientTestDelegates extends SliderTestUtils { describe "containers" - SerializedContainerInformation amContainerInfo2 = + ContainerInformation amContainerInfo2 = appAPI.getContainer(amContainerId) assert amContainerInfo2.containerId == amContainerId @@ -136,17 +135,17 @@ class SliderRestClientTestDelegates extends SliderTestUtils { describe "components" - Map<String, SerializedComponentInformation> components = + Map<String, ComponentInformation> components = appAPI.enumComponents() // two components assert components.size() >= 1 log.info "${components}" - SerializedComponentInformation amComponentInfo = - (SerializedComponentInformation) components[COMPONENT_AM] + ComponentInformation amComponentInfo = + (ComponentInformation) components[COMPONENT_AM] - SerializedComponentInformation amFullInfo = appAPI.getComponent(COMPONENT_AM) + ComponentInformation amFullInfo = appAPI.getComponent(COMPONENT_AM) assert amFullInfo.containers.size() == 1 assert amFullInfo.containers[0] == amContainerId http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/181c5e9f/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy index 4672da9..4fc82b9 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy @@ -19,7 +19,7 @@ package org.apache.slider.server.appmaster.model.appstate import groovy.util.logging.Slf4j -import org.apache.slider.api.types.SerializedContainerInformation +import org.apache.slider.api.types.ContainerInformation import org.apache.slider.core.persist.JsonSerDeser import org.apache.slider.server.appmaster.management.MetricsAndMonitoring import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest @@ -76,8 +76,8 @@ class TestMockAppStateAppRestIntegration extends BaseMockAppStateTest implements map = clr.refresh() assert map.size() == instances.size() log.info("$map") - JsonSerDeser<SerializedContainerInformation> serDeser = - new JsonSerDeser<>(SerializedContainerInformation) + JsonSerDeser<ContainerInformation> serDeser = + new JsonSerDeser<>(ContainerInformation) map.each { key, value -> log.info("$key -> ${serDeser.toJson(value)}") }
