http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java b/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java index c408ed2..7425c2d 100644 --- a/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java +++ b/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java @@ -82,7 +82,8 @@ public class RestTypeMarshalling { if (wire.hasFailureMessage()) { info.failureMessage = wire.getFailureMessage(); } - + info.pendingAntiAffineRequestCount = wire.getPendingAntiAffineRequestCount(); + info.pendingAntiAffineRequest = info.pendingAntiAffineRequestCount > 0; return info; } @@ -132,11 +133,11 @@ public class RestTypeMarshalling { if (info.containers != null) { builder.addAllContainers(info.containers); } + builder.setPendingAntiAffineRequestCount(info.pendingAntiAffineRequestCount); return builder.build(); } - public static ContainerInformation - unmarshall(Messages.ContainerInformationProto wire) { + public static ContainerInformation unmarshall(Messages.ContainerInformationProto wire) { ContainerInformation info = new ContainerInformation(); info.containerId = wire.getContainerId(); info.component = wire.getComponent();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java index 52f6c08..3b4b8bd 100644 --- a/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java +++ b/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java @@ -19,6 +19,7 @@ package org.apache.slider.api.types; import org.apache.slider.api.StatusKeys; +import org.apache.slider.server.appmaster.state.RoleStatus; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.map.annotate.JsonSerialize; @@ -35,13 +36,15 @@ import java.util.Map; * <p> * This means that if any fields are added here. they must be added to * <code>src/main/proto/SliderClusterMessages.proto</code> and - * the probuf structures rebuilt. + * the protobuf structures rebuilt via a {@code mvn generate-sources -Pcompile-protobuf} + * + * See also {@link RoleStatus#serialize()} */ @JsonIgnoreProperties(ignoreUnknown = true) @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) public class ComponentInformation { - + public String name; public int priority; public int desired, actual, releasing; @@ -49,6 +52,9 @@ public class ComponentInformation { public int requested; public int failed, started, startFailed, completed, totalRequested; public int nodeFailed, failedRecently, preempted; + public boolean pendingAntiAffineRequest; + public int pendingAntiAffineRequestCount; + public String failureMessage; public List<String> containers; @@ -59,16 +65,17 @@ public class ComponentInformation { public Map<String, Integer> buildStatistics() { Map<String, Integer> stats = new HashMap<>(); stats.put(StatusKeys.STATISTICS_CONTAINERS_ACTIVE_REQUESTS, requested); + stats.put(StatusKeys.STATISTICS_CONTAINERS_ANTI_AFFINE_PENDING, pendingAntiAffineRequestCount); stats.put(StatusKeys.STATISTICS_CONTAINERS_COMPLETED, completed); stats.put(StatusKeys.STATISTICS_CONTAINERS_DESIRED, desired); stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED, failed); + stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED_NODE, nodeFailed); + stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED_RECENTLY, failedRecently); stats.put(StatusKeys.STATISTICS_CONTAINERS_LIVE, actual); + stats.put(StatusKeys.STATISTICS_CONTAINERS_PREEMPTED, preempted); stats.put(StatusKeys.STATISTICS_CONTAINERS_REQUESTED, totalRequested); stats.put(StatusKeys.STATISTICS_CONTAINERS_STARTED, started); stats.put(StatusKeys.STATISTICS_CONTAINERS_START_FAILED, startFailed); - stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED_RECENTLY, failedRecently); - stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED_NODE, nodeFailed); - stats.put(StatusKeys.STATISTICS_CONTAINERS_PREEMPTED, preempted); return stats; } @@ -76,22 +83,23 @@ public class ComponentInformation { public String toString() { final StringBuilder sb = new StringBuilder("ComponentInformation{"); - sb.append("failureMessage='").append(failureMessage).append('\''); - sb.append(", totalRequested=").append(totalRequested); + sb.append(", name='").append(name).append('\''); + sb.append(", actual=").append(actual); + sb.append(", anti-affine pending").append(pendingAntiAffineRequestCount); sb.append(", completed=").append(completed); - sb.append(", startFailed=").append(startFailed); - sb.append(", started=").append(started); + sb.append(", desired=").append(desired); sb.append(", failed=").append(failed); - sb.append(", requested=").append(requested); + sb.append(", failureMessage='").append(failureMessage).append('\''); sb.append(", placementPolicy=").append(placementPolicy); - sb.append(", releasing=").append(releasing); - sb.append(", actual=").append(actual); - sb.append(", desired=").append(desired); sb.append(", priority=").append(priority); - sb.append(", name='").append(name).append('\''); - sb.append(", container count='") - .append(containers== null ? 0: containers.size()) - .append('\''); + sb.append(", releasing=").append(releasing); + sb.append(", requested=").append(requested); + sb.append(", started=").append(started); + sb.append(", startFailed=").append(startFailed); + sb.append(", totalRequested=").append(totalRequested); + sb.append("container count='") + .append(containers == null ? 0 : containers.size()) + .append('\''); sb.append('}'); return sb.toString(); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java new file mode 100644 index 0000000..33a875e --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.api.types; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +/** + * Serialized node entry information. Must be kept in sync with the protobuf equivalent. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class NodeEntryInformation { + + public int priority; + + /** + * instance explicitly requested on this node: it's OK if an allocation + * comes in that has not been (and when that happens, this count should + * not drop). + */ + public int requested; + + /** number of starting instances */ + public int starting; + + /** incrementing counter of instances that failed to start */ + public int startFailed; + + /** incrementing counter of instances that failed */ + public int failed; + + /** + * Counter of "failed recently" events. These are all failures + * which have happened since it was last reset. + */ + public int failedRecently; + + /** incrementing counter of instances that have been pre-empted. */ + public int preempted; + + /** + * Number of live nodes. + */ + public int live; + + /** number of containers being released off this node */ + public int releasing; + + /** timestamp of last use */ + public long lastUsed; +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java new file mode 100644 index 0000000..842db14 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.api.types; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.util.ArrayList; +import java.util.List; + +/** + * Serialized node information. Must be kept in sync with the protobuf equivalent. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class NodeInformation { + + public String hostname; + public String state; + public String httpAddress; + public String rackName; + public String labels; + public String healthReport; + public long lastUpdated; + public List<NodeEntryInformation> entries = new ArrayList<>(); + + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/AMUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/AMUtils.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/AMUtils.java deleted file mode 100644 index 32684c6..0000000 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/AMUtils.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster; - -import org.apache.slider.common.SliderExitCodes; -import org.apache.slider.core.main.LauncherExitCodes; - -public class AMUtils { - /** - * Map an exit code from a process - * @param exitCode - * @return an exit code - */ - public static int mapProcessExitCodeToYarnExitCode(int exitCode) { - switch (exitCode) { - case LauncherExitCodes.EXIT_SUCCESS: - return LauncherExitCodes.EXIT_SUCCESS; - //remap from a planned shutdown to a failure - case LauncherExitCodes.EXIT_CLIENT_INITIATED_SHUTDOWN: - return SliderExitCodes.EXIT_SUCCESS; - default: - return exitCode; - } - } - - public static boolean isMappedExitAFailure(int mappedExitCode) { - return mappedExitCode != 0; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java new file mode 100644 index 0000000..50b5dad --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.util.Records; +import org.apache.slider.server.appmaster.state.AbstractClusterServices; + +public class ProtobufClusterServices extends AbstractClusterServices { + + public Resource newResource() { + return Records.newRecord(Resource.class); + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufRecordFactory.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufRecordFactory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufRecordFactory.java deleted file mode 100644 index d7f79f1..0000000 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufRecordFactory.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster; - -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.util.Records; -import org.apache.slider.server.appmaster.state.AbstractRecordFactory; - -public class ProtobufRecordFactory extends AbstractRecordFactory { - public Resource newResource() { - return Records.newRecord(Resource.class); - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 019ec71..0e7e295 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -291,7 +291,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService * live on, etc. */ private final AppState appState = - new AppState(new ProtobufRecordFactory(), metricsAndMonitoring); + new AppState(new ProtobufClusterServices(), metricsAndMonitoring); /** * App state for external objects. This is almost entirely @@ -301,7 +301,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService private final ProviderAppState stateForProviders = new ProviderAppState("undefined", appState); - /** * model the state using locks and conditions */ @@ -1000,7 +999,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService .start(webApp); WebAppService<SliderAMWebApp> webAppService = - new WebAppService<SliderAMWebApp>("slider", webApp); + new WebAppService<>("slider", webApp); deployChildService(webAppService); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java index 5b96256..8de27e7 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java @@ -24,7 +24,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.service.CompositeService; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -49,8 +51,14 @@ public class MetricsAndMonitoring extends CompositeService { final HealthCheckRegistry health = new HealthCheckRegistry(); private final Map<String, MeterAndCounter> meterAndCounterMap - = new ConcurrentHashMap<String, MeterAndCounter>(); - + = new ConcurrentHashMap<>(); + + /** + * List of recorded events + */ + private final List<RecordedEvent> eventHistory = new ArrayList<>(100); + + public static final int EVENT_LIMIT = 1000; public MetricRegistry getMetrics() { return metrics; @@ -100,4 +108,23 @@ public class MetricsAndMonitoring extends CompositeService { meter.mark(); } + /** + * Add an event (synchronized) + * @param event event + */ + public synchronized void noteEvent(RecordedEvent event) { + if (eventHistory.size() > EVENT_LIMIT) { + eventHistory.remove(0); + } + eventHistory.add(event); + } + + /** + * Clone the event history; blocks for the duration of the copy operation. + * @return a new list + */ + public synchronized List<RecordedEvent> cloneEventHistory() { + return new ArrayList<>(eventHistory); + } } + http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/management/RecordedEvent.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/RecordedEvent.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/RecordedEvent.java new file mode 100644 index 0000000..d48d337 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/RecordedEvent.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.management; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.text.DateFormat; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class RecordedEvent { + private static final DateFormat dateFormat = DateFormat.getDateInstance(); + public long id; + public String name; + public long timestamp; + public String time; + public String category; + public String host; + public int role; + public String text; + + public RecordedEvent() { + } + + /** + * Create an event. The timestamp is also converted to a time string + * @param id id counter + * @param name event name + * @param timestamp timestamp. If non-zero, is used to build the {@code time} text field. + * @param category even category + * @param text arbitrary text + */ + public RecordedEvent(long id, String name, long timestamp, String category, String text) { + this.id = id; + this.name = name; + this.timestamp = timestamp; + this.time = timestamp > 0 ? dateFormat.format(timestamp) : ""; + this.category = category; + this.text = text; + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java index 594ee47..b7794ed 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java @@ -25,6 +25,11 @@ import org.apache.hadoop.yarn.client.api.AMRMClient; import java.util.List; public interface RMOperationHandlerActions { + + /** + * Release an assigned container + * @param containerId container + */ void releaseAssignedContainer(ContainerId containerId); /** @@ -38,7 +43,7 @@ public interface RMOperationHandlerActions { * @param request request to cancel */ void cancelSingleRequest(AMRMClient.ContainerRequest request); - + /** * Remove a container request * @param priority1 priority to remove at http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java new file mode 100644 index 0000000..27e25f9 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.state; + +import org.apache.hadoop.yarn.api.records.Resource; + +/** + * Cluster services offered by the YARN infrastructure. + */ +public abstract class AbstractClusterServices { + /** + * Create a resource for requests + * @return a resource which can be built up. + */ + public abstract Resource newResource(); + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractRecordFactory.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractRecordFactory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractRecordFactory.java deleted file mode 100644 index e9655f0..0000000 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractRecordFactory.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.state; - -import org.apache.hadoop.yarn.api.records.Resource; - -/** - * Factory supplying records created by the App state; entry point - * for mock code. - */ -public abstract class AbstractRecordFactory { - - public abstract Resource newResource(); -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index 6990a24..0f77824 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -115,7 +115,7 @@ public class AppState { protected static final Logger log = LoggerFactory.getLogger(AppState.class); - private final AbstractRecordFactory recordFactory; + private final AbstractClusterServices recordFactory; private final MetricsAndMonitoring metricsAndMonitoring; @@ -179,13 +179,13 @@ public class AppState { private ClusterDescription clusterStatusTemplate = new ClusterDescription(); private final Map<Integer, RoleStatus> roleStatusMap = - new ConcurrentHashMap<Integer, RoleStatus>(); + new ConcurrentHashMap<>(); private final Map<String, ProviderRole> roles = - new ConcurrentHashMap<String, ProviderRole>(); + new ConcurrentHashMap<>(); private final Map<Integer, ProviderRole> rolePriorityMap = - new ConcurrentHashMap<Integer, ProviderRole>(); + new ConcurrentHashMap<>(); /** * The master node. @@ -265,7 +265,7 @@ public class AppState { * Nodes that came assigned to a role above that * which were asked for -this appears to happen */ - private final Set<ContainerId> surplusNodes = new HashSet<ContainerId>(); + private final Set<ContainerId> surplusNodes = new HashSet<>(); /** * Map of containerID to cluster nodes, for status reports. @@ -308,7 +308,7 @@ public class AppState { * @param recordFactory factory for YARN records * @param metricsAndMonitoring metrics and monitoring services */ - public AppState(AbstractRecordFactory recordFactory, + public AppState(AbstractClusterServices recordFactory, MetricsAndMonitoring metricsAndMonitoring) { this.recordFactory = recordFactory; this.metricsAndMonitoring = metricsAndMonitoring; @@ -1439,6 +1439,10 @@ public class AppState { } } + /** + * Handle node update from the RM. This syncs up the node map with the RM's view + * @param updatedNodes updated nodes + */ public synchronized void onNodesUpdated(List<NodeReport> updatedNodes) { roleHistory.onNodesUpdated(updatedNodes); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java index 0f46054..8ff0895 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java @@ -18,23 +18,24 @@ package org.apache.slider.server.appmaster.state; +import org.apache.slider.api.types.NodeEntryInformation; + /** * Information about the state of a role on a specific node instance. * No fields are synchronized; sync on the instance to work with it - <p> - The two fields `releasing` and `requested` are used to track the ongoing - state of YARN requests; they do not need to be persisted across stop/start - cycles. They may be relevant across AM restart, but without other data - structures in the AM, not enough to track what the AM was up to before - it was restarted. The strategy will be to ignore unexpected allocation - responses (which may come from pre-restart) requests, while treating - unexpected container release responses as failures. - <p> - The `active` counter is only decremented after a container release response - has been received. - <p> - - Accesses are synchronized. + * <p> + * The two fields `releasing` and `requested` are used to track the ongoing + * state of YARN requests; they do not need to be persisted across stop/start + * cycles. They may be relevant across AM restart, but without other data + * structures in the AM, not enough to track what the AM was up to before + * it was restarted. The strategy will be to ignore unexpected allocation + * responses (which may come from pre-restart) requests, while treating + * unexpected container release responses as failures. + * <p> + * The `active` counter is only decremented after a container release response + * has been received. + * <p> + * */ public class NodeEntry { @@ -47,30 +48,45 @@ public class NodeEntry { /** * instance explicitly requested on this node: it's OK if an allocation * comes in that has not been (and when that happens, this count should - * not drop) + * not drop). */ private int requested; + + /** number of starting instances */ private int starting; + + /** incrementing counter of instances that failed to start */ private int startFailed; + + /** incrementing counter of instances that failed */ private int failed; - private int preempted; + /** * Counter of "failed recently" events. These are all failures * which have happened since it was last reset. */ private int failedRecently; + + /** incrementing counter of instances that have been pre-empted. */ + private int preempted; + /** * Number of live nodes. */ private int live; + + /** number of containers being released off this node */ private int releasing; + + /** timestamp of last use */ private long lastUsed; - + /** - * Is the node available for assignments. This does not track - * whether or not there are any outstanding requests for this node - * @return true if there are no role instances here - * other than some being released. + * Is the node available for assignments? That is, it is + * not running any instances of this type, nor are there + * any requests oustanding for it. + * @return true if a new request could be issued without taking + * the number of instances > 1. */ public synchronized boolean isAvailable() { return getActive() == 0 && (requested == 0) && starting == 0; @@ -266,4 +282,23 @@ public class NodeEntry { sb.append('}'); return sb.toString(); } + + /** + * Produced a serialized form which can be served up as JSON + * @return a summary of the current role status. + */ + public synchronized NodeEntryInformation serialize() { + NodeEntryInformation info = new NodeEntryInformation(); + info.priority = rolePriority; + info.requested = requested; + info.releasing = releasing; + info.starting = starting; + info.startFailed = startFailed; + info.failed = failed; + info.failedRecently = failedRecently; + info.preempted = preempted; + info.live = live; + info.lastUsed = lastUsed; + return info; + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java index fb80b5f..2afdc42 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java @@ -18,6 +18,12 @@ package org.apache.slider.server.appmaster.state; +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.slider.api.types.NodeEntryInformation; +import org.apache.slider.api.types.NodeInformation; +import org.apache.slider.common.tools.SliderUtils; + import java.io.Serializable; import java.util.ArrayList; import java.util.Comparator; @@ -33,6 +39,19 @@ public class NodeInstance { public final String hostname; + /** + * last state of node. Starts off as {@link NodeState#RUNNING}, + * on the assumption that it is live. + */ + private NodeState nodeState = NodeState.RUNNING; + + private NodeReport nodeReport = null; + + /** + * time of state update + */ + private long nodeStateUpdateTime = 0; + private final List<NodeEntry> nodeEntries; /** @@ -44,6 +63,20 @@ public class NodeInstance { nodeEntries = new ArrayList<>(roles); } + + /** + * Update the node status + * @param report latest node report + * @return true if the node state changed + */ + public boolean updateNode(NodeReport report) { + nodeReport = report; + NodeState oldState = nodeState; + nodeState = report.getNodeState(); + nodeStateUpdateTime = report.getLastHealthReportTime(); + return nodeState != oldState; + } + /** * Get the entry for a role -if present * @param role role index @@ -215,11 +248,35 @@ public class NodeInstance { } /** + * Produced a serialized form which can be served up as JSON + * @return a summary of the current role status. + */ + public synchronized NodeInformation serialize() { + NodeInformation info = new NodeInformation(); + info.hostname = hostname; + // null-handling state constructor + info.state = "" + nodeState; + info.lastUpdated = nodeStateUpdateTime; + if (nodeReport != null) { + info.httpAddress = nodeReport.getHttpAddress(); + info.rackName = nodeReport.getRackName(); + info.labels = SliderUtils.join(nodeReport.getNodeLabels(), ", ", false); + info.healthReport = nodeReport.getHealthReport(); + } + info.entries = new ArrayList<>(nodeEntries.size()); + for (NodeEntry nodeEntry : nodeEntries) { + info.entries.add(nodeEntry.serialize()); + } + return info; + } + + /** * A comparator for sorting entries where the node is preferred over another. * <p> * The exact algorithm may change * - * @return +ve int if left is preferred to right; -ve if right over left, 0 for equal + * the comparision is a positive int if left is preferred to right; + * negative if right over left, 0 for equal */ public static class Preferred implements Comparator<NodeInstance>, Serializable { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java index aa50baa..b631057 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java @@ -19,6 +19,7 @@ package org.apache.slider.server.appmaster.state; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.yarn.api.records.NodeReport; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,9 +27,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.List; -import java.util.ListIterator; import java.util.Map; /** @@ -84,27 +83,6 @@ public class NodeMap extends HashMap<String, NodeInstance> { } /** - * purge the history of all nodes that have been inactive since the absolute time - * @param absoluteTime time - * @return the number purged - */ - public int purgeUnusedEntries(long absoluteTime) { - int purged = 0; - Iterator<Map.Entry<String, NodeInstance>> iterator = - entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry<String, NodeInstance> entry = iterator.next(); - NodeInstance ni = entry.getValue(); - if (!ni.purgeUnusedEntries(absoluteTime)) { - iterator.remove(); - purged ++; - } - } - return purged; - } - - - /** * reset the failed recently counters */ public void resetFailedRecently() { @@ -114,6 +92,17 @@ public class NodeMap extends HashMap<String, NodeInstance> { } } + + /** + * Update the node state + * @param hostname host name + * @param report latest node report + * @return the updated node. + */ + public boolean updateNode(String hostname, NodeReport report) { + return getOrCreate(hostname).updateNode(report); + } + /** * Clone point * @return http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java index 78fdcc9..f1e26bb 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.slider.common.tools.SliderUtils; @@ -66,14 +67,11 @@ public class RoleHistory { LoggerFactory.getLogger(RoleHistory.class); private final List<ProviderRole> providerRoles; private long startTime; - /** - * Time when saved - */ + + /** Time when saved */ private long saveTime; - /** - * If the history was loaded, the time at which the history was saved - * - */ + + /** If the history was loaded, the time at which the history was saved */ private long thawedDataTime; private NodeMap nodemap; @@ -331,18 +329,6 @@ public class RoleHistory { } /** - * Garbage collect the structure -this will drop - * all nodes that have been inactive since the (relative) age. - * This will drop the failure counts of the nodes too, so it will - * lose information that matters. - * @param age relative age - */ - public void gc(long age) { - long absoluteTime = now() - age; - purgeUnusedEntries(absoluteTime); - } - - /** * Mark ourselves as dirty */ public void touch() { @@ -355,15 +341,6 @@ public class RoleHistory { } /** - * purge the history of - * all nodes that have been inactive since the absolute time - * @param absoluteTime time - */ - public synchronized void purgeUnusedEntries(long absoluteTime) { - nodemap.purgeUnusedEntries(absoluteTime); - } - - /** * reset the failed recently counters */ public synchronized void resetFailedRecently() { @@ -451,7 +428,7 @@ public class RoleHistory { if (loadedRoleHistory != null) { rebuild(loadedRoleHistory); thawSuccessful = true; - Path loadPath = loadedRoleHistory.getPath();; + Path loadPath = loadedRoleHistory.getPath(); log.debug("loaded history from {}", loadPath); // delete any old entries try { @@ -627,8 +604,8 @@ public class RoleHistory { } /** - * Get the list of active nodes ... walks the node map so - * is O(nodes) + * Get the list of active nodes ... walks the node map so + * is {@code O(nodes)} * @param role role index * @return a possibly empty list of nodes with an instance of that node */ @@ -654,8 +631,7 @@ public class RoleHistory { * @throws RuntimeException if the container has no hostname */ public synchronized NodeInstance getOrCreateNodeInstance(Container container) { - String hostname = RoleHistoryUtils.hostnameOf(container); - return nodemap.getOrCreate(hostname); + return nodemap.getOrCreate(RoleHistoryUtils.hostnameOf(container)); } /** @@ -738,7 +714,7 @@ public class RoleHistory { } /** - * Event: a container start has been submitter + * Event: a container start has been submitted * @param container container being started * @param instance instance bound to the container */ @@ -775,19 +751,25 @@ public class RoleHistory { * @param updatedNodes list of updated nodes */ public synchronized void onNodesUpdated(List<NodeReport> updatedNodes) { + log.debug("Updating {} nodes", updatedNodes.size()); for (NodeReport updatedNode : updatedNodes) { - String hostname = updatedNode.getNodeId() == null + String hostname = updatedNode.getNodeId() == null ? null : updatedNode.getNodeId().getHost(); - if (hostname == null) { + NodeState nodeState = updatedNode.getNodeState(); + if (hostname == null || nodeState == null) { continue; } - if (updatedNode.getNodeState() != null - && updatedNode.getNodeState().isUnusable()) { - failedNodes.add(hostname); - nodemap.remove(hostname); - } else { - failedNodes.remove(hostname); + // update the node; this also creates an instance if needed + boolean updated = nodemap.updateNode(hostname, updatedNode); + if (updated) { + log.debug("Updated host {} to state {}", hostname, nodeState); + if (nodeState.isUnusable()) { + log.info("Failed node {}", hostname); + failedNodes.add(hostname); + } else { + failedNodes.remove(hostname); + } } } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java index e2974fc..98a8311 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java @@ -25,6 +25,7 @@ import org.apache.slider.providers.ProviderRole; import java.io.Serializable; import java.util.Comparator; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -51,6 +52,15 @@ public final class RoleStatus implements Cloneable { private final AtomicLong failedRecently = new AtomicLong(0); private final AtomicLong limitsExceeded = new AtomicLong(0); + /** flag set to true if there is an outstanding anti-affine request */ + private final AtomicBoolean pendingAARequest = new AtomicBoolean(false); + + /** + * Number of AA requests queued. These should be reduced first on a + * flex down. + */ + private int pendingAntiAffineRequestCount = 0; + private String failureMessage = ""; public RoleStatus(ProviderRole providerRole) { @@ -302,6 +312,7 @@ public final class RoleStatus implements Cloneable { ", actual=" + actual + ", requested=" + requested + ", releasing=" + releasing + + ", pendingAntiAffineRequestCount=" + pendingAntiAffineRequestCount + ", failed=" + failed + ", failed recently=" + failedRecently.get() + ", node failed=" + nodeFailed.get() + @@ -355,6 +366,8 @@ public final class RoleStatus implements Cloneable { info.failedRecently = failedRecently.intValue(); info.nodeFailed = nodeFailed.intValue(); info.preempted = preempted.intValue(); + info.pendingAntiAffineRequest = pendingAARequest.get(); + info.pendingAntiAffineRequestCount = pendingAntiAffineRequestCount; return info; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java index eef178a..56c7cac 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java @@ -37,7 +37,8 @@ import java.util.List; import java.util.Map; /** - * The methods to offer state access to the providers + * The methods to offer state access to the providers and other parts of + * the system which want read-only access to the state. */ public interface StateAccessForProviders { @@ -280,5 +281,10 @@ public interface StateAccessForProviders { */ List<RoleInstance> lookupRoleContainers(String component); + /** + * Get the JSON serializable information about a component + * @param component component to look up + * @return a structure describing the component. + */ ComponentInformation getComponentInformation(String component); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java index 179ae8c..0f99d6d 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java @@ -81,7 +81,15 @@ public interface WebAppApi { */ QueueAccess getQueues(); + /** + * API for AM operations + * @return current operations implementation + */ AppMasterActionOperations getAMOperations(); + /** + * Local cache of content + * @return the cache + */ ContentCache getContentCache(); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/proto/SliderClusterMessages.proto ---------------------------------------------------------------------- diff --git a/slider-core/src/main/proto/SliderClusterMessages.proto b/slider-core/src/main/proto/SliderClusterMessages.proto index caca87b..8287f36 100644 --- a/slider-core/src/main/proto/SliderClusterMessages.proto +++ b/slider-core/src/main/proto/SliderClusterMessages.proto @@ -24,21 +24,28 @@ package org.apache.slider.api; //import "Security.proto"; +/* + To debug compilation problems, bypass the maven build and invoke protoc + from the command line + + protoc --java_out=target src/main/proto/SliderClusterMessages.proto +*/ + message RoleInstanceState { - required string name = 1; - optional string role = 2; - required uint32 state = 4; - required uint32 exitCode = 5; - optional string command = 6; + required string name = 1; + optional string role = 2; + required uint32 state = 4; + required uint32 exitCode = 5; + optional string command = 6; optional string diagnostics = 7; - repeated string output = 8; + repeated string output = 8; repeated string environment = 9; - required uint32 roleId = 10; - required bool released = 11; - required int64 createTime = 12; - required int64 startTime = 13; - required string host = 14; - required string hostURL = 15; + required uint32 roleId = 10; + required bool released = 11; + required int64 createTime = 12; + required int64 startTime = 13; + required string host = 14; + required string hostURL = 15; optional string appVersion = 16; } @@ -65,9 +72,9 @@ message UpgradeContainersRequestProto { /** message to include */ - required string message = 1; - repeated string container = 2; - repeated string component = 3; + required string message = 1; + repeated string container = 2; + repeated string component = 3; } /** @@ -181,9 +188,9 @@ message KillContainerResponseProto { * AM suicide */ message AMSuicideRequestProto { - required string text = 1; - required int32 signal = 2; - required int32 delay = 3; + required string text = 1; + required int32 signal = 2; + required int32 delay = 3; } /** @@ -206,9 +213,9 @@ message GetInstanceDefinitionRequestProto { * Get the definition back as three separate JSON strings */ message GetInstanceDefinitionResponseProto { - required string internal = 1; - required string resources = 2; - required string application = 3; + required string internal = 1; + required string resources = 2; + required string application = 3; } @@ -233,42 +240,43 @@ message ApplicationLivenessInformationProto { * see org.apache.slider.api.types.ComponentInformation */ message ComponentInformationProto { - optional string name = 1; - optional int32 priority = 2; - optional int32 desired = 3; - optional int32 actual = 4; - optional int32 releasing = 5; - optional int32 requested = 6; - optional int32 failed = 7; - optional int32 started = 8; - optional int32 startFailed = 9; - optional int32 completed = 10; - optional int32 totalRequested = 11; - optional string failureMessage = 12 ; - optional int32 placementPolicy = 13; - repeated string containers = 14; + optional string name = 1; + optional int32 priority = 2; + optional int32 desired = 3; + optional int32 actual = 4; + optional int32 releasing = 5; + optional int32 requested = 6; + optional int32 failed = 7; + optional int32 started = 8; + optional int32 startFailed = 9; + optional int32 completed = 10; + optional int32 totalRequested = 11; + optional string failureMessage =12; + optional int32 placementPolicy =13; + repeated string containers = 14; optional int32 failedRecently = 15; - optional int32 nodeFailed = 16; - optional int32 preempted = 17; + optional int32 nodeFailed = 16; + optional int32 preempted = 17; + optional int32 pendingAntiAffineRequestCount = 18; } /* * see org.apache.slider.api.types.ContainerInformation */ message ContainerInformationProto { - optional string containerId = 1; - optional string component = 2; - optional bool released = 3; - optional int32 state = 4; - optional int32 exitCode = 5; - optional string diagnostics = 6; - optional int64 createTime = 7; - optional int64 startTime = 8; - repeated string output = 9; - optional string host = 10; - optional string hostURL = 11; - optional string placement = 12; - optional string appVersion = 13; + optional string containerId = 1; + optional string component = 2; + optional bool released = 3; + optional int32 state = 4; + optional int32 exitCode = 5; + optional string diagnostics = 6; + optional int64 createTime = 7; + optional int64 startTime = 8; + repeated string output = 9; + optional string host = 10; + optional string hostURL = 11; + optional string placement = 12; + optional string appVersion = 13; } @@ -279,10 +287,32 @@ message PingInformationProto { optional string text = 1; optional string verb = 2; optional string body = 3; - optional int64 time = 4; + optional int64 time = 4; } +message NodeEntryInformationProto { + required int32 priority = 1; + required int32 requested = 2; + required int32 starting = 3; + required int32 startFailed = 4; + required int32 failed = 5; + required int32 failedRecently= 6; + required int32 preempted = 7; + required int32 live = 8; + required int32 releasing = 9; + required int64 lastUsed = 10; +} +message NodeInformationProto { + required string hostname = 1; + required string state = 2; + required string httpAddress = 3; + required string rackName = 4; + required string labels = 5; + required string healthReport= 6; + required int64 lastUpdated = 7; + repeated NodeEntryInformationProto entries = 8; +} message GetModelRequestProto { } @@ -345,12 +375,12 @@ message WrappedJsonProto { } message GetCertificateStoreRequestProto { - optional string hostname = 1; + optional string hostname = 1; required string requesterId = 2; - required string password = 3; - required string type = 4; + required string password = 3; + required string type = 4; } message GetCertificateStoreResponseProto { required bytes store = 1; -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy index 2cc13c2..6e21a38 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy @@ -19,7 +19,7 @@ package org.apache.slider.server.appmaster.model.mock import org.apache.slider.providers.ProviderRole import org.apache.slider.server.appmaster.management.MetricsAndMonitoring -import org.apache.slider.server.appmaster.state.AbstractRecordFactory +import org.apache.slider.server.appmaster.state.AbstractClusterServices import org.apache.slider.server.appmaster.state.AppState /** @@ -27,7 +27,7 @@ import org.apache.slider.server.appmaster.state.AppState */ class MockAppState extends AppState { - public MockAppState(AbstractRecordFactory recordFactory) { + public MockAppState(AbstractClusterServices recordFactory) { super(recordFactory, new MetricsAndMonitoring()); } @@ -37,7 +37,7 @@ class MockAppState extends AppState { * Instance with a mock record factory */ public MockAppState() { - super(new MockRecordFactory(), new MetricsAndMonitoring()); + super(new MockClusterServices(), new MetricsAndMonitoring()); } public Map<String, ProviderRole> getRoleMap() { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy new file mode 100644 index 0000000..d27a6bb --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.mock + +import org.apache.hadoop.yarn.api.records.Resource +import org.apache.slider.server.appmaster.state.AbstractClusterServices + +class MockClusterServices extends AbstractClusterServices { + + @Override + Resource newResource() { + return new MockResource() + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy deleted file mode 100644 index f7d353f..0000000 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.mock - -import org.apache.hadoop.yarn.api.records.Resource -import org.apache.slider.server.appmaster.state.AbstractRecordFactory - -class MockRecordFactory extends AbstractRecordFactory { - - @Override - Resource newResource() { - return new MockResource() - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy index e1964f5..5a7a891 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy @@ -19,15 +19,14 @@ package org.apache.slider.server.appmaster.web.view import com.google.inject.AbstractModule import com.google.inject.Guice import com.google.inject.Injector -import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.hadoop.yarn.webapp.hamlet.Hamlet import org.apache.slider.api.ClusterDescription import org.apache.slider.providers.ProviderService import org.apache.slider.server.appmaster.model.mock.MockAppState import org.apache.slider.server.appmaster.model.mock.MockProviderService -import org.apache.slider.server.appmaster.model.mock.MockRecordFactory -import org.apache.slider.server.appmaster.state.AbstractRecordFactory +import org.apache.slider.server.appmaster.model.mock.MockClusterServices +import org.apache.slider.server.appmaster.state.AbstractClusterServices import org.apache.slider.server.appmaster.state.AppState import org.apache.slider.server.appmaster.state.ProviderAppState import org.apache.slider.server.appmaster.web.WebAppApi @@ -43,7 +42,7 @@ public class TestClusterSpecificationBlock { @Before public void setup() { - AppState appState = new MyAppState(new MockRecordFactory()); + AppState appState = new MyAppState(new MockClusterServices()); ProviderAppState providerAppState = new ProviderAppState( "undefined", appState) @@ -79,7 +78,7 @@ public class TestClusterSpecificationBlock { } private static class MyAppState extends MockAppState { - public MyAppState(AbstractRecordFactory recordFactory) { + public MyAppState(AbstractClusterServices recordFactory) { super(recordFactory); this.clusterStatus = new MockClusterDescription(); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java index d37fcea..4c43168 100644 --- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java +++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java @@ -37,7 +37,7 @@ import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; import org.apache.slider.server.appmaster.model.mock.MockFactory; import org.apache.slider.server.appmaster.model.mock.MockProviderService; -import org.apache.slider.server.appmaster.model.mock.MockRecordFactory; +import org.apache.slider.server.appmaster.model.mock.MockClusterServices; import org.apache.slider.server.appmaster.state.AppState; import org.apache.slider.server.appmaster.state.ProviderAppState; import org.apache.slider.server.appmaster.state.SimpleReleaseSelector; @@ -126,7 +126,7 @@ public class TestAMAgentWebServices { historyPath = new org.apache.hadoop.fs.Path(historyWorkDir.toURI()); fs.delete(historyPath, true); - appState = new AppState(new MockRecordFactory(), new MetricsAndMonitoring()); + appState = new AppState(new MockClusterServices(), new MetricsAndMonitoring()); appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES); appState.buildInstance( factory.newInstanceDefinition(0, 0, 0), http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java index 7cb9395..df7e002 100644 --- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java +++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java @@ -42,7 +42,7 @@ import org.apache.slider.core.persist.JsonSerDeser; import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; import org.apache.slider.server.appmaster.model.mock.MockFactory; import org.apache.slider.server.appmaster.model.mock.MockProviderService; -import org.apache.slider.server.appmaster.model.mock.MockRecordFactory; +import org.apache.slider.server.appmaster.model.mock.MockClusterServices; import org.apache.slider.server.appmaster.state.AppState; import org.apache.slider.server.appmaster.state.ProviderAppState; import org.apache.slider.server.appmaster.state.SimpleReleaseSelector; @@ -167,7 +167,7 @@ public class TestAMManagementWebServices extends JerseyTest { historyPath = new org.apache.hadoop.fs.Path(historyWorkDir.toURI()); fs.delete(historyPath, true); - appState = new AppState(new MockRecordFactory(), new MetricsAndMonitoring()); + appState = new AppState(new MockClusterServices(), new MetricsAndMonitoring()); appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES); appState.buildInstance( factory.newInstanceDefinition(0, 0, 0),