Github user srdo commented on a diff in the pull request: https://github.com/apache/storm/pull/1642#discussion_r78969293 --- Diff: storm-core/src/jvm/org/apache/storm/daemon/supervisor/Slot.java --- @@ -0,0 +1,776 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.storm.daemon.supervisor; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.storm.Config; +import org.apache.storm.cluster.IStormClusterState; +import org.apache.storm.generated.ExecutorInfo; +import org.apache.storm.generated.LSWorkerHeartbeat; +import org.apache.storm.generated.LocalAssignment; +import org.apache.storm.generated.ProfileAction; +import org.apache.storm.generated.ProfileRequest; +import org.apache.storm.localizer.ILocalizer; +import org.apache.storm.scheduler.ISupervisor; +import org.apache.storm.utils.LocalState; +import org.apache.storm.utils.Time; +import org.apache.storm.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Slot extends Thread implements AutoCloseable { + private static final Logger LOG = LoggerFactory.getLogger(Slot.class); + + static enum MachineState { + EMPTY, + RUNNING, + WAITING_FOR_WORKER_START, + KILL_AND_RELAUNCH, + KILL, + WAITING_FOR_BASIC_LOCALIZATION, + WAITING_FOR_BLOB_LOCALIZATION; + }; + + static class StaticState { + public final ILocalizer localizer; + public final long hbTimeoutMs; + public final long firstHbTimeoutMs; + public final long killSleepMs; + public final long monitorFreqMs; + public final ContainerLauncher containerLauncher; + public final int port; + public final String host; + public final ISupervisor iSupervisor; + public final LocalState localState; + + StaticState(ILocalizer localizer, long hbTimeoutMs, long firstHbTimeoutMs, + long killSleepMs, long monitorFreqMs, + ContainerLauncher containerLauncher, String host, int port, + ISupervisor iSupervisor, LocalState localState) { + this.localizer = localizer; + this.hbTimeoutMs = hbTimeoutMs; + this.firstHbTimeoutMs = firstHbTimeoutMs; + this.containerLauncher = containerLauncher; + this.killSleepMs = killSleepMs; + this.monitorFreqMs = monitorFreqMs; + this.host = host; + this.port = port; + this.iSupervisor = iSupervisor; + this.localState = localState; + } + } + + static class DynamicState { + public final MachineState state; + public final LocalAssignment newAssignment; + public final LocalAssignment currentAssignment; + public final Container container; + public final LocalAssignment pendingLocalization; + public final Future<Void> pendingDownload; + public final Set<TopoProfileAction> profileActions; + public final Set<TopoProfileAction> pendingStopProfileActions; + + /** + * The last time that WAITING_FOR_WORKER_START, KILL, or KILL_AND_RELAUNCH were entered into. + */ + public final long startTime; + + public DynamicState(final LocalAssignment currentAssignment, Container container, final LocalAssignment newAssignment) { + this.currentAssignment = currentAssignment; + this.container = container; + if ((currentAssignment == null) ^ (container == null)) { + throw new IllegalArgumentException("Container and current assignment must both be null, or neither can be null"); + } + + if (currentAssignment == null) { + state = MachineState.EMPTY; + } else { + state = MachineState.RUNNING; + } + + this.startTime = System.currentTimeMillis(); + this.newAssignment = newAssignment; + this.pendingLocalization = null; + this.pendingDownload = null; + this.profileActions = new HashSet<>(); + this.pendingStopProfileActions = new HashSet<>(); + } + + public DynamicState(final MachineState state, final LocalAssignment newAssignment, + final Container container, final LocalAssignment currentAssignment, + final LocalAssignment pendingLocalization, final long startTime, + final Future<Void> pendingDownload, final Set<TopoProfileAction> profileActions, + final Set<TopoProfileAction> pendingStopProfileActions) { + this.state = state; + this.newAssignment = newAssignment; + this.currentAssignment = currentAssignment; + this.container = container; + this.pendingLocalization = pendingLocalization; + this.startTime = startTime; + this.pendingDownload = pendingDownload; + this.profileActions = profileActions; + this.pendingStopProfileActions = pendingStopProfileActions; + } + + public String toString() { + StringBuffer sb = new StringBuffer(); + sb.append(state); + sb.append(" msInState: "); + sb.append(Time.currentTimeMillis() - startTime); + if (container != null) { + sb.append(" "); + sb.append(container); + } + return sb.toString(); + } + + public DynamicState withNewAssignment(LocalAssignment newAssignment) { + return new DynamicState(this.state, newAssignment, + this.container, this.currentAssignment, + this.pendingLocalization, this.startTime, + this.pendingDownload, this.profileActions, + this.pendingStopProfileActions); + } + + public DynamicState withPendingLocalization(LocalAssignment pendingLocalization, Future<Void> pendingDownload) { + return new DynamicState(this.state, this.newAssignment, + this.container, this.currentAssignment, + pendingLocalization, this.startTime, + pendingDownload, this.profileActions, + this.pendingStopProfileActions); + } + + public DynamicState withPendingLocalization(Future<Void> pendingDownload) { + return withPendingLocalization(this.pendingLocalization, pendingDownload); + } + + public DynamicState withState(final MachineState state) { + long newStartTime = Time.currentTimeMillis(); + return new DynamicState(state, this.newAssignment, + this.container, this.currentAssignment, + this.pendingLocalization, newStartTime, + this.pendingDownload, this.profileActions, + this.pendingStopProfileActions); + } + + public DynamicState withCurrentAssignment(final Container container, final LocalAssignment currentAssignment) { + return new DynamicState(this.state, this.newAssignment, + container, currentAssignment, + this.pendingLocalization, this.startTime, + this.pendingDownload, this.profileActions, + this.pendingStopProfileActions); + } + + public DynamicState withProfileActions(Set<TopoProfileAction> profileActions, Set<TopoProfileAction> pendingStopProfileActions) { + return new DynamicState(this.state, this.newAssignment, + this.container, this.currentAssignment, + this.pendingLocalization, this.startTime, + this.pendingDownload, profileActions, + pendingStopProfileActions); + } + }; + + static class TopoProfileAction { + public final String topoId; + public final ProfileRequest request; + + public TopoProfileAction(String topoId, ProfileRequest request) { + this.topoId = topoId; + this.request = request; + } + + @Override + public int hashCode() { + return (37 * topoId.hashCode()) + request.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof TopoProfileAction)) { + return false; + } + TopoProfileAction o = (TopoProfileAction) other; + return topoId.equals(o.topoId) && request.equals(o.request); + } + + @Override + public String toString() { + return "{ " + topoId + ": " + request + " }"; + } + } + + static boolean equivalent(LocalAssignment a, LocalAssignment b) { + if (a == null && b == null) { + return true; + } + if (a != null && b != null) { + if (a.get_topology_id().equals(b.get_topology_id())) { + Set<ExecutorInfo> aexec = new HashSet<>(a.get_executors()); + Set<ExecutorInfo> bexec = new HashSet<>(b.get_executors()); + if (aexec.equals(bexec)) { + boolean aHasResources = a.is_set_resources(); + boolean bHasResources = b.is_set_resources(); + if (!aHasResources && !bHasResources) { + return true; + } + if (aHasResources && bHasResources) { + if (a.get_resources().equals(b.get_resources())) { + return true; + } + } + } + } + } + return false; + } + + static DynamicState stateMachineStep(DynamicState dynamicState, StaticState staticState) throws Exception { + LOG.debug("STATE {}", dynamicState.state); + switch (dynamicState.state) { + case EMPTY: + return handleEmpty(dynamicState, staticState); + case RUNNING: + return handleRunning(dynamicState, staticState); + case WAITING_FOR_WORKER_START: + return handleWaitingForWorkerStart(dynamicState, staticState); + case KILL_AND_RELAUNCH: + return handleKillAndRelaunch(dynamicState, staticState); + case KILL: + return handleKill(dynamicState, staticState); + case WAITING_FOR_BASIC_LOCALIZATION: + return handleWaitingForBasicLocalization(dynamicState, staticState); + case WAITING_FOR_BLOB_LOCALIZATION: + return handleWaitingForBlobLocalization(dynamicState, staticState); + default: + throw new IllegalStateException("Code not ready to handle a state of "+dynamicState.state); + } + } + + /** + * Prepare for a new assignment my downloading new required blobs, or going to empty if there is nothing to download. + * PRECONDITION: The slot should be empty + * @param dynamicState current state + * @param staticState static data + * @return the next state + * @throws IOException on any error + */ + static DynamicState prepareForNewAssignmentOnEmptySlot(DynamicState dynamicState, StaticState staticState) throws IOException { + assert(dynamicState.container == null); --- End diff -- Maybe there should be an assertion that slot state is empty?
--- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---