Repository: incubator-slider Updated Branches: refs/heads/develop 22ffc648c -> 6b33fd892
SLIDER-713 flex REST command. This is failing in a test against the REST endpoint Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a06f53ac Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a06f53ac Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a06f53ac Branch: refs/heads/develop Commit: a06f53ac3a10a8040c7b5fbaf757456c19cb4d78 Parents: 9e58112 Author: Steve Loughran <ste...@apache.org> Authored: Tue Feb 17 18:20:16 2015 +0000 Committer: Steve Loughran <ste...@apache.org> Committed: Tue Feb 17 18:20:16 2015 +0000 ---------------------------------------------------------------------- .../apache/slider/api/SliderApplicationApi.java | 9 +++++ .../slider/api/SliderClusterProtocol.java | 2 +- .../client/ipc/SliderApplicationIpcClient.java | 12 ++++++ .../client/ipc/SliderClusterOperations.java | 27 +++++++++++-- .../slider/client/rest/BaseRestClient.java | 2 +- .../rest/SliderApplicationApiRestClient.java | 35 ++++++++++++++++ .../apache/slider/common/SliderXmlConfKeys.java | 3 +- .../rpc/SliderClusterProtocolProxy.java | 5 +-- .../server/appmaster/rpc/SliderIPCService.java | 2 +- .../rest/application/ApplicationResource.java | 42 +++++++++++++++++++- .../rest/AbstractAppApiTestDelegates.groovy | 22 ++++++++++ 11 files changed, 149 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a06f53ac/slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java b/slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java index 6b591fd..7de1915 100644 --- a/slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java +++ b/slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java @@ -23,6 +23,7 @@ import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.api.types.ContainerInformation; import org.apache.slider.api.types.PingInformation; import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.core.conf.ConfTree; import org.apache.slider.core.conf.ConfTreeOperations; import java.io.IOException; @@ -57,6 +58,14 @@ public interface SliderApplicationApi { ConfTreeOperations getDesiredResources() throws IOException; /** + * Put an updated resources structure. This triggers a cluster flex + * operation + * @param updated updated resources + * @throws IOException on any problem. + */ + void putDesiredResources(ConfTree updated) throws IOException; + + /** * Get the aggregate resolved model * @return the aggregate configuration of what was asked for * âafter resolution has taken place http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a06f53ac/slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java b/slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java index 7babab7..aa3dd3a 100644 --- a/slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java +++ b/slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java @@ -49,7 +49,7 @@ public interface SliderClusterProtocol extends VersionedProtocol { * Flex the cluster. */ Messages.FlexClusterResponseProto flexCluster(Messages.FlexClusterRequestProto request) - throws IOException, YarnException; + throws IOException; /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a06f53ac/slider-core/src/main/java/org/apache/slider/client/ipc/SliderApplicationIpcClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/ipc/SliderApplicationIpcClient.java b/slider-core/src/main/java/org/apache/slider/client/ipc/SliderApplicationIpcClient.java index 13a2e03..bada24e 100644 --- a/slider-core/src/main/java/org/apache/slider/client/ipc/SliderApplicationIpcClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/ipc/SliderApplicationIpcClient.java @@ -26,6 +26,7 @@ import org.apache.slider.api.types.ContainerInformation; import org.apache.slider.api.types.PingInformation; import org.apache.slider.api.SliderApplicationApi; import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.core.conf.ConfTree; import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.core.exceptions.NoSuchNodeException; import org.slf4j.Logger; @@ -109,6 +110,17 @@ public class SliderApplicationIpcClient implements SliderApplicationApi { } } + + @Override + public void putDesiredResources(ConfTree updated) throws IOException { + try { + operations.flex(updated); + } catch (IOException e) { + throw convert(e); + } + } + + @Override public AggregateConf getResolvedModel() throws IOException { try { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a06f53ac/slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java b/slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java index faaf619..2952b18 100644 --- a/slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java +++ b/slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java @@ -92,6 +92,12 @@ public class SliderClusterOperations { return ClusterNode.fromProtobuf(node.getClusterNode()); } + /** + * Unmarshall a list of nodes from a protobud response + * @param nodes node list + * @return possibly empty list of cluster nodes + * @throws IOException + */ public List<ClusterNode> convertNodeWireToClusterNodes(List<Messages.RoleInstanceState> nodes) throws IOException { List<ClusterNode> nodeList = new ArrayList<ClusterNode>(nodes.size()); @@ -144,6 +150,14 @@ public class SliderClusterOperations { } } + /** + * Get the AM instance definition. + * <p> + * See {@link SliderClusterProtocol#getInstanceDefinition(Messages.GetInstanceDefinitionRequestProto)} + * @return current slider AM aggregate definition + * @throws YarnException + * @throws IOException + */ public AggregateConf getInstanceDefinition() throws YarnException, IOException { Messages.GetInstanceDefinitionRequestProto.Builder builder = @@ -308,8 +322,14 @@ public class SliderClusterOperations { } return state; } - - public boolean flex(ConfTree resources) throws IOException, YarnException { + + /** + * Flex operation + * @param resources new resources + * @return the response + * @throws IOException + */ + public boolean flex(ConfTree resources) throws IOException { Messages.FlexClusterRequestProto request = Messages.FlexClusterRequestProto.newBuilder() .setClusterSpec(resources.toJson()) @@ -358,8 +378,7 @@ public class SliderClusterOperations { } - - + public AggregateConf getModelDesired() throws IOException { return unmarshallToAggregateConf(appMaster.getModelDesired(EMPTY)); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a06f53ac/slider-core/src/main/java/org/apache/slider/client/rest/BaseRestClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/rest/BaseRestClient.java b/slider-core/src/main/java/org/apache/slider/client/rest/BaseRestClient.java index de2fe20..c21844d 100644 --- a/slider-core/src/main/java/org/apache/slider/client/rest/BaseRestClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/rest/BaseRestClient.java @@ -76,7 +76,7 @@ public class BaseRestClient { try { Preconditions.checkArgument(c != null); resource.accept(MediaType.APPLICATION_JSON_TYPE); - return (T) resource.method(method.getVerb(), c); + return resource.method(method.getVerb(), c); } catch (ClientHandlerException ex) { throw ExceptionConverter.convertJerseyException(method.getVerb(), resource.getURI().toString(), http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a06f53ac/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java b/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java index d1c81b0..a6eb5e3 100644 --- a/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java @@ -20,7 +20,10 @@ package org.apache.slider.client.rest; import com.google.common.base.Preconditions; import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientHandlerException; +import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.GenericType; +import com.sun.jersey.api.client.UniformInterfaceException; import com.sun.jersey.api.client.WebResource; import com.sun.jersey.api.representation.Form; import org.apache.commons.lang.StringUtils; @@ -31,6 +34,7 @@ import org.apache.slider.api.SliderApplicationApi; import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTree; import org.apache.slider.core.conf.ConfTreeOperations; +import org.apache.slider.core.exceptions.ExceptionConverter; import org.apache.slider.core.restclient.HttpVerb; import org.apache.slider.api.types.PingInformation; import org.slf4j.Logger; @@ -164,6 +168,23 @@ public class SliderApplicationApiRestClient extends BaseRestClient } @Override + public void putDesiredResources(ConfTree updated) throws IOException { + WebResource resource = applicationResource(MODEL_DESIRED_RESOURCES); + try { + resource.entity(updated, MediaType.APPLICATION_JSON_TYPE); +// resource.put(ConfTree.class, updated); + resource.put(ClientResponse.class, updated); + } catch (ClientHandlerException ex) { + throw ExceptionConverter.convertJerseyException("PUT", + resource.getURI().toString(), + ex); + } catch (UniformInterfaceException ex) { + throw ExceptionConverter.convertJerseyException("PUT", + resource.getURI().toString(), ex); + } + } + + @Override public AggregateConf getResolvedModel() throws IOException { return getApplicationResource(MODEL_RESOLVED, AggregateConf.class); } @@ -250,6 +271,20 @@ public class SliderApplicationApiRestClient extends BaseRestClient f.add("text", text); return pingResource.post(PingInformation.class, f); } + + /** + * Ping as a POST + * @param text text to include + * @return the response + * @throws IOException on any failure + */ + public PingInformation pingPut(String text) throws IOException { + WebResource pingResource = applicationResource(ACTION_PING); + pingResource.type(MediaType.APPLICATION_JSON_TYPE); + Form f = new Form(); + f.add("text", text); + return pingResource.post(PingInformation.class, f); + } @Override public void stop(String text) throws IOException { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a06f53ac/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java index a808058..b337a1a 100644 --- a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java +++ b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java @@ -149,8 +149,9 @@ public interface SliderXmlConfKeys { /** * Flag to indicate the insecure AM filter is enabled by default: {@value}. + * THIS IS CURRENTLY TRUE: NOT FOR RELEASE */ - boolean X_DEV_INSECURE_DEFAULT = false; + boolean X_DEV_INSECURE_DEFAULT = true; /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a06f53ac/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java index 7ecbbb6..d7031a5 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java @@ -100,9 +100,8 @@ public class SliderClusterProtocolProxy implements SliderClusterProtocol { } @Override - public Messages.FlexClusterResponseProto flexCluster(Messages.FlexClusterRequestProto request) throws - IOException, - YarnException { + public Messages.FlexClusterResponseProto flexCluster(Messages.FlexClusterRequestProto request) + throws IOException { try { return endpoint.flexCluster(NULL_CONTROLLER, request); } catch (ServiceException e) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a06f53ac/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java index 882c963..a590c05 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java @@ -181,7 +181,7 @@ public class SliderIPCService extends AbstractService @Override //SliderClusterProtocol public Messages.FlexClusterResponseProto flexCluster(Messages.FlexClusterRequestProto request) - throws IOException, YarnException { + throws IOException { onRpcCall("flex"); String payload = request.getClusterSpec(); ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a06f53ac/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java index 2dedfb8..9eea539 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java @@ -27,6 +27,9 @@ import org.apache.slider.api.types.ContainerInformation; import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTree; import org.apache.slider.core.exceptions.NoSuchNodeException; +import org.apache.slider.server.appmaster.actions.ActionFlexCluster; +import org.apache.slider.server.appmaster.actions.AsyncAction; +import org.apache.slider.server.appmaster.actions.QueueAccess; import org.apache.slider.server.appmaster.state.RoleInstance; import org.apache.slider.server.appmaster.state.StateAccessForProviders; import org.apache.slider.server.appmaster.web.WebAppApi; @@ -56,10 +59,12 @@ import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Context; import static javax.ws.rs.core.MediaType.*; + +import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; -import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; @Singleton @SuppressWarnings("unchecked") @@ -88,11 +93,13 @@ public class ApplicationResource extends AbstractSliderResource { */ private final ContentCache cache; private final StateAccessForProviders state; + private final QueueAccess actionQueues; public ApplicationResource(WebAppApi slider) { super(slider); state = slider.getAppState(); cache = slider.getContentCache(); + actionQueues = slider.getQueues(); } /** @@ -148,6 +155,23 @@ public class ApplicationResource extends AbstractSliderResource { return lookupConfTree(MODEL_DESIRED_RESOURCES); } + @PUT + @Path(MODEL_DESIRED_RESOURCES) + @Produces({APPLICATION_JSON}) + @Consumes({APPLICATION_JSON}) + public ConfTree putModelDesiredResources( + ConfTree updated) { + markPut(SLIDER_SUBPATH_APPLICATION, MODEL_DESIRED_RESOURCES); + log.info("PUT {}:\n{}", MODEL_DESIRED_RESOURCES, + updated); + queue(new ActionFlexCluster("flex", + 1, TimeUnit.MILLISECONDS, + updated)); + // return the updated value, even though it potentially hasn't yet + // been executed + return updated; + } + @GET @Path(MODEL_RESOLVED) @Produces({APPLICATION_JSON}) @@ -402,4 +426,20 @@ TODO: decide what structure to return here, then implement return new RestActionStop(slider).stop(request, uriInfo, body); } + /** + * Schedule an action + * @param action for delayed execution + */ + public void schedule(AsyncAction action) { + actionQueues.schedule(action); + } + + /** + * Put an action on the immediate queue -to be executed when the queue + * reaches it. + * @param action action to queue + */ + public void queue(AsyncAction action) { + actionQueues.put(action); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a06f53ac/slider-core/src/test/groovy/org/apache/slider/agent/rest/AbstractAppApiTestDelegates.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/AbstractAppApiTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/AbstractAppApiTestDelegates.groovy index aaa0fd6..724a4d2 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/AbstractAppApiTestDelegates.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/AbstractAppApiTestDelegates.groovy @@ -227,9 +227,31 @@ public abstract class AbstractAppApiTestDelegates extends AbstractRestTestDelega testAppLiveness() } + public void testFlexOperation() { + // no-op flex; get current state and push out again. + def current = appAPI.getDesiredResources() + + def uuid = UUID.randomUUID() + current.set("yarn.test.noop", uuid) + appAPI.putDesiredResources(current.confTree) + + + + } + + /** + * Get the resolved value and push that out as the new state + * + */ + public void testFlexToResolved() { + def resolved = appAPI.getResolvedResources() + appAPI.putDesiredResources(resolved.confTree) + } + @Override public void testSuiteComplexVerbs() { testPing(); + testFlexOperation(); } }
