Repository: incubator-slider Updated Branches: refs/heads/develop a2c266126 -> ff7b8c37f
SLIDER-539 Web view to list outstanding container requests Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ff7b8c37 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ff7b8c37 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ff7b8c37 Branch: refs/heads/develop Commit: ff7b8c37f22e82ea36b18c22703a007bb67c5511 Parents: a2c2661 Author: Steve Loughran <ste...@apache.org> Authored: Sun Oct 19 14:20:23 2014 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Sun Oct 19 14:20:23 2014 +0100 ---------------------------------------------------------------------- .../slider/server/appmaster/state/AppState.java | 17 +++++ .../appmaster/state/ProviderAppState.java | 4 + .../server/appmaster/state/RoleHistory.java | 6 +- .../server/appmaster/state/RoleStatus.java | 25 +++++++ .../state/StateAccessForProviders.java | 3 +- .../server/appmaster/web/view/IndexBlock.java | 78 ++++++++++++++------ .../appmaster/web/view/TestIndexBlock.groovy | 32 +++++++- 7 files changed, 136 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ff7b8c37/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index 834eaf2..21cb6d2 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -787,6 +787,23 @@ public class AppState { return lookupRoleStatus(ContainerPriority.extractRole(c)); } + /** + * Get a clone of the role status list. Concurrent events may mean this + * list (or indeed, some of the role status entries) may be inconsistent + * @return a snapshot of the role status entries + */ + public List<RoleStatus> cloneRoleStatusList() { + Collection<RoleStatus> statuses = roleStatusMap.values(); + List<RoleStatus> statusList = new ArrayList<RoleStatus>(statuses.size()); + try { + for (RoleStatus status : statuses) { + statusList.add((RoleStatus)(status.clone())); + } + } catch (CloneNotSupportedException e) { + log.warn("Unexpected cloning failure: {}", e, e); + } + return statusList; + } public RoleStatus lookupRoleStatus(String name) throws YarnRuntimeException { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ff7b8c37/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java index 9c5da12..d5a041b 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java @@ -211,4 +211,8 @@ public class ProviderAppState implements StateAccessForProviders { appState.refreshClusterStatus(); } + @Override + public List<RoleStatus> cloneRoleStatusList() { + return appState.cloneRoleStatusList(); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ff7b8c37/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java index dca7384..01fedc6 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java @@ -350,7 +350,7 @@ public class RoleHistory { * Handler for bootstrap event */ public void onBootstrap() { - log.info("Role history bootstrapped"); + log.debug("Role history bootstrapped"); } /** @@ -371,7 +371,7 @@ public class RoleHistory { } if (loaded != null) { thawSuccessful = true; - log.info("loaded history from {}", loaded); + log.debug("loaded history from {}", loaded); // delete any old entries try { int count = historyWriter.purgeOlderHistoryEntries(filesystem, loaded); @@ -469,7 +469,7 @@ public class RoleHistory { List<NodeInstance> targets = getNodesForRoleId(roleKey); int cnt = targets == null ? 0 : targets.size(); - log.info("There're {} nodes to consider for {}", cnt, role.getName()); + log.debug("There're {} nodes to consider for {}", cnt, role.getName()); while (targets != null && !targets.isEmpty() && nodeInstance == null) { NodeInstance head = targets.remove(0); if (head.getActiveRoleInstances(roleKey) == 0) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ff7b8c37/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java index 63b5931..f9feca3 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java @@ -22,6 +22,8 @@ import org.apache.slider.api.StatusKeys; import org.apache.slider.providers.PlacementPolicy; import org.apache.slider.providers.ProviderRole; +import java.io.Serializable; +import java.util.Comparator; import java.util.HashMap; import java.util.Map; @@ -298,4 +300,27 @@ public final class RoleStatus implements Cloneable { stats.put(StatusKeys.STATISTICS_CONTAINERS_START_FAILED, getStartFailed()); return stats; } + + /** + * Compare two role status entries by name + */ + public static class CompareByName implements Comparator<RoleStatus>, + Serializable { + @Override + public int compare(RoleStatus o1, RoleStatus o2) { + return o1.getName().compareTo(o2.getName()); + } + } + + /** + * Compare two role status entries by key + */ + public static class CompareByKey implements Comparator<RoleStatus>, + Serializable { + @Override + public int compare(RoleStatus o1, RoleStatus o2) { + return Integer.compare(o1.getKey(), o2.getKey()); + } + } + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ff7b8c37/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java index b907b06..75076ed 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java @@ -201,9 +201,8 @@ public interface StateAccessForProviders { /** * Update the cluster description with anything interesting - * @param providerStatus status from the provider for the cluster info section */ void refreshClusterStatus(); - + List<RoleStatus> cloneRoleStatusList(); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ff7b8c37/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java index 54bdb09..91c70ee 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java @@ -16,20 +16,25 @@ */ package org.apache.slider.server.appmaster.web.view; +import com.google.common.annotations.VisibleForTesting; import com.google.inject.Inject; -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.UL; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; +import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.StatusKeys; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.providers.ProviderService; +import org.apache.slider.server.appmaster.state.RoleStatus; import org.apache.slider.server.appmaster.state.StateAccessForProviders; import org.apache.slider.server.appmaster.web.WebAppApi; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collections; +import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Map.Entry; @@ -37,15 +42,14 @@ import java.util.Map.Entry; * */ public class IndexBlock extends HtmlBlock { - private static final String HBASE = "HBase"; private static final Logger log = LoggerFactory.getLogger(IndexBlock.class); - private StateAccessForProviders appState; + private StateAccessForProviders appView; private ProviderService providerService; @Inject public IndexBlock(WebAppApi slider) { - this.appState = slider.getAppState(); + this.appView = slider.getAppState(); this.providerService = slider.getProviderService(); } @@ -57,45 +61,73 @@ public class IndexBlock extends HtmlBlock { } // An extra method to make testing easier since you can't make an instance of Block + @VisibleForTesting protected void doIndex(Hamlet html, String providerName) { - DIV<Hamlet> div = html.div("general_info").h1("index_header", providerName + " cluster: '" + appState.getClusterStatus().name + "'"); + ClusterDescription clusterStatus = appView.getClusterStatus(); + DIV<Hamlet> div = html.div("general_info") + .h1("index_header", + "Application: '" + clusterStatus.name + "'"); UL<DIV<Hamlet>> ul = div.ul(); - ul.li("Total number of containers for cluster: " + appState.getNumOwnedContainers()); - ul.li("Cluster created: " + getInfoAvoidingNulls(StatusKeys.INFO_CREATE_TIME_HUMAN)); - ul.li("Cluster last flexed: " + getInfoAvoidingNulls(StatusKeys.INFO_FLEX_TIME_HUMAN)); - ul.li("Cluster running since: " + getInfoAvoidingNulls(StatusKeys.INFO_LIVE_TIME_HUMAN)); - ul.li("Cluster HDFS storage path: " + appState.getClusterStatus().dataPath); - ul.li("Cluster configuration path: " + appState.getClusterStatus().originConfigurationPath); + ul.li("Total number of containers for application: " + appView.getNumOwnedContainers()); + ul.li("Application created: " + + getInfoAvoidingNulls(StatusKeys.INFO_CREATE_TIME_HUMAN)); + ul.li("Application last flexed: " + getInfoAvoidingNulls(StatusKeys.INFO_FLEX_TIME_HUMAN)); + ul.li("Application running since: " + getInfoAvoidingNulls(StatusKeys.INFO_LIVE_TIME_HUMAN)); + ul.li("Application HDFS storage path: " + clusterStatus.dataPath); + ul.li("Application configuration path: " + clusterStatus.originConfigurationPath); ul._()._(); html.div("provider_info").h3(providerName + " specific information"); ul = div.ul(); - addProviderServiceOptions(providerService, ul); + addProviderServiceOptions(providerService, ul, clusterStatus); ul._()._(); - } - - private String getProviderName() { - String providerServiceName = providerService.getName().toLowerCase(); - // Get HBase properly capitalized - if (providerServiceName.contains("hbase")) { - return HBASE; + html.div("container_instances").h3("Component Instances"); + + Hamlet.TABLE<DIV<Hamlet>> table = div.table(); + table.tr() + .th("Component") + .th("Desired") + .th("Actual") + .th("Requested") + .th("Failed") + .th("Failed to start") + ._(); + + List<RoleStatus> roleStatuses = appView.cloneRoleStatusList(); + Collections.sort(roleStatuses, new RoleStatus.CompareByName()); + for (RoleStatus status : roleStatuses) { + table.tr() + .td(status.getName()) + .th(String.format("%d", status.getDesired())) + .th(String.format("%d", status.getActual())) + .th(String.format("%d", status.getRequested())) + .th(String.format("%d", status.getFailed())) + .th(String.format("%d", status.getStartFailed())) + ._(); } - return StringUtils.capitalize(providerServiceName); + table._()._(); + } + + private String getProviderName() { + String providerServiceName = providerService.getName().toLowerCase(Locale.ENGLISH); + return providerServiceName; } private String getInfoAvoidingNulls(String key) { - String createTime = appState.getClusterStatus().getInfo(key); + String createTime = appView.getClusterStatus().getInfo(key); return null == createTime ? "N/A" : createTime; } - protected void addProviderServiceOptions(ProviderService providerService, UL<DIV<Hamlet>> ul) { - Map<String, String> details = providerService.buildMonitorDetails(appState.getClusterStatus()); + protected void addProviderServiceOptions(ProviderService providerService, + UL<DIV<Hamlet>> ul, ClusterDescription clusterStatus) { + Map<String, String> details = providerService.buildMonitorDetails( + clusterStatus); if (null == details) { return; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ff7b8c37/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy index a3a4118..d232ecb 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy @@ -82,6 +82,20 @@ public class TestIndexBlock extends BaseMockAppStateTest { @Test public void testIndex() { + def role0 = role0Status + def role1 = role1Status + role0.desired = 8 + role0.incActual() + role0.incActual() + role0.incActual() + role0.incActual() + role0.incActual() + role1.incRequested() + role1.incRequested() + role1.incRequested() + role0.noteFailed(false, "") + role0.noteFailed(true, "") + StringWriter sw = new StringWriter(64); PrintWriter pw = new PrintWriter(sw); @@ -89,7 +103,23 @@ public class TestIndexBlock extends BaseMockAppStateTest { int level = hamlet.nestLevel(); indexBlock.doIndex(hamlet, "accumulo"); + + def body = sw.toString() + log.info(body) + assertEquals(body, level, hamlet.nestLevel()) + // verify role data came out + assert body.contains("role0") + // + assert body.contains("8") + assert body.contains("5") + assert body.contains("3") + assert body.contains("2") + assert body.contains("1") - assert level == hamlet.nestLevel(); + assert body.contains("role1") + assert body.contains("role2") + // verify that the sorting took place + assert body.indexOf("role0") < body.indexOf("role1") + assert body.indexOf("role1") < body.indexOf("role2") } } \ No newline at end of file