SLIDER-149 REST registry interface; in sync with latest changes of the code
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/da814b74 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/da814b74 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/da814b74 Branch: refs/heads/develop Commit: da814b7440ea8aeb21b9811df53b93b0eddece33 Parents: 44e6165 Author: Steve Loughran <ste...@apache.org> Authored: Thu Sep 25 20:43:50 2014 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Thu Sep 25 20:43:50 2014 +0100 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 11 +- .../core/exceptions/ExceptionConverter.java | 16 +- .../docstore/PublishedConfiguration.java | 4 +- .../core/registry/info/RegisteredDocument.java | 33 --- .../core/registry/info/RegisteredEndpoint.java | 1 + .../core/registry/info/RegistryFields.java | 34 --- .../core/registry/info/RegistryNaming.java | 51 ----- .../slider/core/registry/info/RegistryView.java | 1 + .../server/appmaster/SliderAppMaster.java | 21 +- .../web/rest/registry/PathEntryResource.java | 51 +++++ .../web/rest/registry/RegistryResource.java | 81 ++++--- .../server/services/curator/CuratorHelper.java | 1 + .../TestStandaloneYarnRegistryAM.groovy | 24 ++- .../registry/curator/TestLocalRegistry.groovy | 164 -------------- .../curator/TestRegistryRestResources.groovy | 195 ----------------- .../curator/TestServiceInstanceSerDeser.groovy | 122 ----------- .../rest/registry/PathEntryMarshalling.groovy | 28 +++ .../registry/TestRegistryRestMarshalling.groovy | 47 ++++ .../registry/TestRegistryRestResources.groovy | 213 +++++++++++++++++++ slider-core/src/test/resources/log4j.properties | 3 +- .../appmaster/web/rest/registry/sample.json | 76 +++++++ .../funtest/FunctionalHBaseClusterIT.groovy | 7 +- .../minicluster/live/TestHBaseMaster.groovy | 2 +- 23 files changed, 526 insertions(+), 660 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index e68a7cb..de27dcd 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.registry.client.api.RegistryOperations; import static org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils.*; import org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils; +import org.apache.hadoop.yarn.registry.client.exceptions.NoRecordException; import org.apache.hadoop.yarn.registry.client.types.Endpoint; import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus; import org.apache.hadoop.yarn.registry.client.types.ServiceRecord; @@ -2453,9 +2454,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe private void logInstance(ServiceRecord instance, boolean verbose) { if (!verbose) { - log.info("{}", instance.id); + log.info("{}", instance.yarn_id); } else { - log.info("{}: ", instance.id); + log.info("{}: ", instance.yarn_id); logEndpoints(instance); } } @@ -2599,7 +2600,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * @param serviceType service type * @param id instance ID * @return instance data - * @throws UnknownApplicationInstanceException no match + * @throws UnknownApplicationInstanceException no path or service record + * at the end of the path * @throws SliderException other failures * @throws IOException IO problems or wrapped exceptions */ @@ -2609,8 +2611,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return getRegistryOperations().resolve( servicePath(currentUser(), serviceType, id)); + // TODO JDK7 SWITCH } catch (PathNotFoundException e) { throw new UnknownApplicationInstanceException(e.getPath().toString(), e); + } catch (NoRecordException e) { + throw new UnknownApplicationInstanceException(e.getPath().toString(), e); } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java index 98dc028..ed5ceb2 100644 --- a/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java +++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java @@ -20,6 +20,7 @@ package org.apache.slider.core.exceptions; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.UniformInterfaceException; +import org.apache.hadoop.fs.PathAccessDeniedException; import java.io.FileNotFoundException; import java.io.IOException; @@ -39,17 +40,22 @@ public class ExceptionConverter { public static IOException convertJerseyException(String targetURL, UniformInterfaceException exception) { + IOException ioe = null; ClientResponse response = exception.getResponse(); if (response != null) { int status = response.getStatus(); + if (status == 401) { + ioe = new PathAccessDeniedException(targetURL); + } if (status >= 400 && status < 500) { - FileNotFoundException fnfe = - new FileNotFoundException(targetURL); - fnfe.initCause(exception); - return fnfe; + ioe = new FileNotFoundException(targetURL); } } - return new IOException("Failed to GET " + targetURL + ": " + exception, exception); + if (ioe == null) { + ioe = new IOException("Failed to GET " + targetURL + ": " + exception); + } + ioe.initCause(exception); + return ioe; } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java index f76b93b..cbc46f0 100644 --- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java +++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java @@ -150,8 +150,8 @@ public class PublishedConfiguration { /** * Return the values as json string - * @return - * @throws IOException + * @return the JSON representation + * @throws IOException marshalling failure */ public String asJson() throws IOException { ObjectMapper mapper = new ObjectMapper(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredDocument.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredDocument.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredDocument.java deleted file mode 100644 index cde282b..0000000 --- a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredDocument.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.registry.info; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -/** - * A registry document - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -public class RegisteredDocument { - public String contentType; - public String url; - public String description; -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java index 8c065e2..2351e06 100644 --- a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java +++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java @@ -32,6 +32,7 @@ import java.net.URL; @JsonIgnoreProperties(ignoreUnknown = true) @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@Deprecated public class RegisteredEndpoint { // standard types http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryFields.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryFields.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryFields.java deleted file mode 100644 index 441a7de..0000000 --- a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryFields.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.registry.info; - -/** - * Fieldnames in registry entries - */ -public class RegistryFields { - - public static final String ENDPOINTS = "endpoints"; - public static final String INTERNAL_VIEW = "internalView"; - public static final String EXTERNAL_VIEW = "externalView"; - - public static final String PROTOCOL = "protocol"; - public static final String VALUE = "value"; - public static final String DESCRIPTION = "description"; - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryNaming.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryNaming.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryNaming.java deleted file mode 100644 index bd5cf27..0000000 --- a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryNaming.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.registry.info; - -import java.util.Locale; - -/** - * Logic for greating names from registry entries; lets the policy be changed - * later - */ -public class RegistryNaming { - - public static String SLIDER_INSTANCE_NAME_FORMAT = "%2$s"; - - public static String createRegistryServiceType(String instanceName, - String userName, - String serviceName) { - return serviceName; - } - - public static String createRegistryName(String instanceName, - String userName, - String serviceName, - int appId) { - return String.format(Locale.ENGLISH, - SLIDER_INSTANCE_NAME_FORMAT, - userName, - instanceName, - serviceName, - appId); - - } - - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java index bdf70a2..89b9c6e 100644 --- a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java +++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java @@ -26,6 +26,7 @@ import java.util.Map; @JsonIgnoreProperties(ignoreUnknown = true) @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@Deprecated public class RegistryView { /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index df54cfa..2d3e5e4 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -95,7 +95,6 @@ import org.apache.slider.core.main.RunService; import org.apache.slider.core.main.ServiceLauncher; import org.apache.slider.core.persist.ConfTreeSerDeser; import org.apache.slider.core.registry.info.CustomRegistryConstants; -import org.apache.slider.core.registry.info.RegistryNaming; import org.apache.slider.providers.ProviderCompleted; import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderService; @@ -887,7 +886,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService * @throws Exception */ private void registerServiceInstance(String instanceName, - ApplicationId appid) throws Exception { + ApplicationId appid) throws IOException { // the registry is running, so register services @@ -896,14 +895,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService URL agentStatusURI = new URL(agentStatusUrl); String serviceName = SliderKeys.APP_TYPE; int id = appid.getId(); - String serviceType = RegistryNaming.createRegistryServiceType( - instanceName, - service_user_name, - serviceName); - String registryId = - RegistryNaming.createRegistryName(instanceName, service_user_name, - serviceName, id); - //Give the provider restricted access to the state, registry setupInitialRegistryPaths(); @@ -916,9 +907,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService // Yarn registry ServiceRecord serviceRecord = new ServiceRecord(); - String serviceID = appid.toString(); - serviceRecord.id = serviceID; - serviceRecord.persistence = PersistencePolicies.APPLICATION; + serviceRecord.yarn_id = appid.toString(); + serviceRecord.yarn_persistence = PersistencePolicies.APPLICATION; serviceRecord.description = "Slider Application Master"; serviceRecord.addExternalEndpoint( @@ -939,7 +929,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService agentStatusURI, serviceRecord); - // store for clients log.info("Service Record \n{}", serviceRecord); yarnRegistryOperations.putService(service_user_name, @@ -951,8 +940,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService // and a shorter lived binding to the app String attempt = appAttemptID.toString(); ServiceRecord attemptRecord = new ServiceRecord(serviceRecord); - attemptRecord.id = attempt; - attemptRecord.persistence = PersistencePolicies.APPLICATION_ATTEMPT; + attemptRecord.yarn_id = attempt; + attemptRecord.yarn_persistence = PersistencePolicies.APPLICATION_ATTEMPT; yarnRegistryOperations.putComponent( RegistryPathUtils.encodeYarnID(attempt), serviceRecord); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java new file mode 100644 index 0000000..8790871 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.web.rest.registry; + +import org.apache.commons.logging.Log; +import org.apache.hadoop.yarn.registry.client.api.RegistryOperations; +import org.apache.hadoop.yarn.registry.client.exceptions.InvalidRecordException; +import org.apache.hadoop.yarn.registry.client.types.ServiceRecord; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.List; + +/** + * Representation of a path entry + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class PathEntryResource { + + /** + * Child nodes: as the full path to each element + */ + public List<String> nodes; + + /** + * Service record: if null âthere is no resolvable service + * record at this node. + */ + public ServiceRecord service; + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java index 67efa0d..23f7261 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java @@ -23,9 +23,12 @@ import org.apache.hadoop.fs.PathAccessDeniedException; import org.apache.hadoop.fs.PathNotFoundException; import org.apache.hadoop.yarn.registry.client.api.RegistryOperations; import org.apache.hadoop.yarn.registry.client.exceptions.AuthenticationFailedException; +import org.apache.hadoop.yarn.registry.client.exceptions.InvalidRecordException; +import org.apache.hadoop.yarn.registry.client.exceptions.NoRecordException; import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus; import org.apache.hadoop.yarn.webapp.ForbiddenException; import org.apache.hadoop.yarn.webapp.NotFoundException; +import org.apache.hadoop.yarn.webapp.WebAppException; import org.apache.slider.server.appmaster.web.WebAppApi; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,10 +39,12 @@ import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; +import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; +import java.io.IOException; import java.util.List; /** @@ -53,7 +58,6 @@ import java.util.List; * */ @Singleton -//@Path(RestPaths.SLIDER_PATH_REGISTRY) public class RegistryResource { protected static final Logger log = LoggerFactory.getLogger(RegistryResource.class); @@ -80,51 +84,80 @@ public class RegistryResource { } @GET - public Response getRoot(@Context HttpServletRequest request) { - return Response.ok("registry root").build(); + @Produces({MediaType.APPLICATION_JSON}) + public PathEntryResource getRoot(@Context HttpServletRequest request, + @Context UriInfo uriInfo) { + return lookup("/", request, uriInfo); } - // {path:.*} @Path(SERVICE_PATH) @GET @Produces({MediaType.APPLICATION_JSON}) - public Response lookup( + public PathEntryResource lookup( @PathParam("path") String path, @Context HttpServletRequest request, @Context UriInfo uriInfo) { - try { init(request, uriInfo); - List<RegistryPathStatus> list = registry.listFull(path); - return Response.ok("found").build(); + return resolvePath(path); + } + + /** + * Do the actual processing of requests to responses; can be directly + * invoked for testing. + * @param path path to query + * @return the entry + * @throws WebApplicationException on any failure. + */ + public PathEntryResource resolvePath(String path) throws + WebApplicationException { + try { + PathEntryResource pathEntry = + fromRegistry(path); + if (log.isDebugEnabled()) { + log.debug("Resolved:\n{}", pathEntry); + } + return pathEntry; + } catch (WebApplicationException e) { + // rethrow direct + throw e; } catch (PathNotFoundException e) { - throw new NotFoundException(path); + throw new NotFoundException("Not found: " + path); } catch (AuthenticationFailedException e) { throw new ForbiddenException(path); } catch (PathAccessDeniedException e) { throw new ForbiddenException(path); } catch (Exception e) { - return fromException(e); + log.error("Error during generation of response: {}", e, e); + throw new WebApplicationException(e); } } + /** - * Handle an exception - * @param e exception - * @return a response to return + * Build from the registry, filling up the children and service records. + * If there is no service record at the end of the path, that entry is + * null + * @param registry registry operations + * @param path path to query + * @return the built up record + * @throws IOException problems + * */ - Response fromException(Exception e) { - log.error("Error during generation of response: {}", e, e); - if (e instanceof PathNotFoundException) { - return Response.status(Response.Status.NOT_FOUND).build(); - } - if (e instanceof AuthenticationFailedException - || e instanceof PathAccessDeniedException) { - return Response.status(Response.Status.FORBIDDEN).build(); + private PathEntryResource fromRegistry(String path) throws IOException { + PathEntryResource entry = new PathEntryResource(); + try { + entry.service = registry.resolve(path); + } catch (NoRecordException e) { + // ignoring + log.debug("No record at {}", path); + } catch (InvalidRecordException e) { + // swallowing this exception, the sign of "no entry present" + // "nothing parseable" + log.warn("Failed to resolve {}: {}", path, e, e); } - return Response.serverError().build(); + entry.nodes = registry.list(path); + return entry; } - - } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java index 9e6f53a..96d296d 100644 --- a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java +++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java @@ -37,6 +37,7 @@ import org.apache.slider.server.services.registry.SliderRegistryService; * This class creates a curator -but does not start or close it. That * is the responsbility of the owner */ +@Deprecated public class CuratorHelper extends Configured { private final CuratorFramework curator; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy index 65bd077..4cd662b 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy @@ -49,6 +49,12 @@ import org.apache.slider.server.appmaster.PublishedArtifacts import org.apache.slider.server.appmaster.web.rest.RestPaths import org.junit.Test +import static org.apache.slider.core.registry.info.CustomRegistryConstants.AGENT_ONEWAY_REST_API +import static org.apache.slider.core.registry.info.CustomRegistryConstants.AGENT_SECURE_REST_API +import static org.apache.slider.core.registry.info.CustomRegistryConstants.AM_IPC_PROTOCOL +import static org.apache.slider.core.registry.info.CustomRegistryConstants.MANAGEMENT_REST_API +import static org.apache.slider.core.registry.info.CustomRegistryConstants.PUBLISHER_REST_API + /** * work with a YARN registry */ @@ -61,7 +67,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase { public static final String ARTIFACT_NAME = PublishedArtifacts.COMPLETE_CONFIG @Test - public void testYarnRegistryAM() throws Throwable { + public void testStandaloneYarnRegistryAM() throws Throwable { describe "create a masterless AM then perform YARN registry operations on it" @@ -156,7 +162,19 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase { def serviceRecord = serviceRecords[0] log.info(serviceRecord.toString()) - assert 2 <= serviceRecord.external.size() + assert serviceRecord.yarn_id != null; + def externalEndpoints = serviceRecord.external; + assert externalEndpoints.size() > 0 + + def am_ipc_protocol = AM_IPC_PROTOCOL + serviceRecord.getExternalEndpoint(am_ipc_protocol) + assert null != am_ipc_protocol; + + assert null != serviceRecord.getExternalEndpoint(MANAGEMENT_REST_API) + assert null != serviceRecord.getExternalEndpoint(PUBLISHER_REST_API) + // internals + assert null != serviceRecord.getInternalEndpoint(AGENT_ONEWAY_REST_API) + assert null != serviceRecord.getInternalEndpoint(AGENT_SECURE_REST_API) // hit the registry web page def registryEndpoint = serviceRecord.getExternalEndpoint( @@ -311,7 +329,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase { describe registryArgs.toString() def listedInstance = client.actionRegistryListYarn(registryArgs) - assert listedInstance[0].id == serviceRecord.id + assert listedInstance[0].yarn_id == serviceRecord.yarn_id // listconf http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestLocalRegistry.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestLocalRegistry.groovy b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestLocalRegistry.groovy deleted file mode 100644 index 71496d6..0000000 --- a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestLocalRegistry.groovy +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.registry.curator - -import org.apache.hadoop.yarn.conf.YarnConfiguration -import org.apache.slider.common.SliderKeys -import org.apache.slider.core.registry.info.ServiceInstanceData -import org.apache.slider.core.registry.retrieve.RegistryRetriever -import org.apache.slider.server.services.curator.CuratorHelper -import org.apache.slider.server.services.curator.RegistryBinderService -import org.apache.slider.core.registry.info.RegistryNaming -import org.apache.slider.server.services.registry.SliderRegistryService -import org.apache.slider.test.MicroZKCluster -import org.apache.slider.test.SliderTestUtils -import org.junit.After -import org.junit.Before -import org.junit.Test - -class TestLocalRegistry { - MicroZKCluster miniZK - SliderRegistryService registryBinder - - @Before - void setup() { - miniZK = new MicroZKCluster() - miniZK.createCluster() - - registryBinder = createRegistry() - } - - def SliderRegistryService createRegistry( ) { - def conf = new YarnConfiguration() - CuratorHelper curatorHelper = - new CuratorHelper(conf, miniZK.zkBindingString); - - SliderRegistryService registry - registry = curatorHelper.createRegistryBinderService("/services"); - registry.init(conf) - registry.start() - return registry - } - - @After - void teardown() { - registryBinder?.stop() - miniZK?.close() - } - - - @Test - public void testRegisterAndQuery() throws Throwable { - registryBinder.register(SliderKeys.APP_TYPE, "instance3", - new URL("http", "localhost", 80, "/"), - null) - def instance = registryBinder.queryForInstance( - SliderKeys.APP_TYPE, - "instance3") - assert instance != null - } - - @Test - public void testRegisterAndList() throws Throwable { - registryBinder.register(SliderKeys.APP_TYPE, "instance3", - new URL("http", "localhost", 80, "/"), - null) - registryBinder.register(SliderKeys.APP_TYPE, "instance2", - new URL("http", "localhost", 8090, "/"), - null) - def instances = registryBinder.instanceIDs(SliderKeys.APP_TYPE) - assert instances.size() ==2 - def instance = registryBinder.queryForInstance( - SliderKeys.APP_TYPE, - "instance3") - assert instance != null - } - - @Test - public void testMultipleRegistryBinders() throws Throwable { - registryBinder.register(SliderKeys.APP_TYPE, "instance3", - new URL("http", "localhost", 80, "/"), - null) - registryBinder.register(SliderKeys.APP_TYPE, "instance2", - new URL("http", "localhost", 8090, "/"), - null) - SliderRegistryService registry2 = createRegistry() - SliderRegistryService registry3 = createRegistry() - try { - def instances = registry3.instanceIDs(SliderKeys.APP_TYPE) - assert instances.size() == 2 - def instance = registryBinder.queryForInstance( - SliderKeys.APP_TYPE, - "instance3") - assert instance.id == "instance3" - assert instance.name == SliderKeys.APP_TYPE - } finally { - registry3.stop() - registry2.stop() - } - } - - @Test - public void testNamingPolicy() throws Throwable { - - String hobbitName = RegistryNaming.createRegistryServiceType("hobbiton", - "bilbo", - SliderKeys.APP_TYPE); - String hobbitId = - RegistryNaming.createRegistryName( - "hobbiton", - "bilbo", - SliderKeys.APP_TYPE, - 1); - String mordorName = RegistryNaming.createRegistryServiceType("mordor", - "bilbo", - SliderKeys.APP_TYPE); - String mordorId = - RegistryNaming.createRegistryName( - "mordor", - "bilbo", - SliderKeys.APP_TYPE, - 1); - - // service have same name - assert hobbitName == mordorName; - assert mordorId != hobbitId; - registryBinder.register(mordorName, mordorId, - new URL("http", "localhost", 8090, "/"), - new ServiceInstanceData()) - registryBinder.register(hobbitName, hobbitId, - new URL("http", "localhost", 80, "/mordor"), - new ServiceInstanceData()) - def mordorInstance = registryBinder.queryForInstance( - mordorName, - mordorId) - assert mordorInstance.port == 8090 - - def instances = registryBinder.listInstances(SliderKeys.APP_TYPE); - SliderTestUtils.dumpRegistryInstances(instances) - assert instances.size() == 2 - def instance1 = instances[0] - def payload1 = instance1.payload - - // now set up a registry retriever - RegistryRetriever retriever = new RegistryRetriever(payload1) - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy deleted file mode 100644 index 0b77adc..0000000 --- a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.registry.curator - -import com.sun.jersey.api.client.Client -import com.sun.jersey.api.client.ClientResponse -import com.sun.jersey.api.client.UniformInterfaceException -import com.sun.jersey.api.client.WebResource -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import org.apache.curator.x.discovery.ServiceType -import org.apache.hadoop.security.UserGroupInformation -import org.apache.slider.api.StatusKeys -import org.apache.slider.client.SliderClient -import org.apache.slider.common.SliderKeys -import org.apache.slider.core.main.ServiceLauncher -import org.apache.slider.core.registry.info.ServiceInstanceData -import org.apache.slider.providers.agent.AgentTestBase -import org.apache.slider.server.appmaster.web.rest.RestPaths -import org.apache.slider.server.services.curator.CuratorServiceInstance -import org.apache.slider.server.services.curator.CuratorServiceInstances -import org.apache.slider.core.registry.info.RegistryNaming -import org.junit.Test - -import javax.ws.rs.core.MediaType - -import static org.apache.slider.common.params.Arguments.ARG_OPTION -import static org.apache.slider.providers.agent.AgentKeys.* -import static org.apache.slider.providers.agent.AgentTestUtils.createTestClient - -@CompileStatic -@Slf4j -class TestRegistryRestResources extends AgentTestBase { - - public static final String REGISTRY_URI = RestPaths.SLIDER_PATH_REGISTRY; - public static final String WADL = "vnd.sun.wadl+xml" - public static final String CLUSTERNAME = "testregistryws" - - - private String id(String instanceName) { - - RegistryNaming.createRegistryName( - instanceName, - UserGroupInformation.getCurrentUser().getUserName(), - SliderKeys.APP_TYPE, - 1); - } - - - @Test - public void testRestURIs() throws Throwable { - def clustername = CLUSTERNAME - createMiniCluster( - clustername, - configuration, - 1, - 1, - 1, - true, - false) - Map<String, Integer> roles = [:] - File slider_core = new File(new File(".").absoluteFile, "src/test/python"); - File app_def_path = new File(app_def_pkg_path) - String agt_ver = "version" - File agt_ver_path = new File(slider_core, agt_ver) - String agt_conf = "agent.ini" - File agt_conf_path = new File(slider_core, agt_conf) - assert app_def_path.exists() - assert agt_ver_path.exists() - assert agt_conf_path.exists() - ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername, - roles, - [ - ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath, - ARG_OPTION, APP_DEF, toURIArg(app_def_path), - ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path), - ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path), - ], - true, true, - true) - SliderClient sliderClient = launcher.service - def report = waitForClusterLive(sliderClient) - def trackingUrl = report.trackingUrl - log.info("tracking URL is $trackingUrl") - def registry_url = appendToURL(trackingUrl, REGISTRY_URI) - - - def status = dumpClusterStatus(sliderClient, "agent AM") - def liveURL = status.getInfo(StatusKeys.INFO_AM_WEB_URL) - if (liveURL) { - registry_url = appendToURL(liveURL, REGISTRY_URI) - - } - - log.info("Registry is $registry_url") - log.info("stacks is ${liveURL}stacks") - log.info("conf is ${liveURL}conf") - - - //WS get - Client client = createTestClient(); - - WebResource webResource = client.resource(registry_url); - List<String> serviceList = webResource.type(MediaType.APPLICATION_JSON) - .get(List.class); - log.info("service list: {}", serviceList) - assert serviceList.contains(SliderKeys.APP_TYPE) - - // test the available GET URIs - webResource = client.resource( - appendToURL(registry_url, RestPaths.REGISTRY_SERVICE)); - - ClientResponse response = webResource.type(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); - def responseStr = response.getEntity(String.class) - log.info("response is " + responseStr) - - "{\"names\":[\"${SliderKeys.APP_TYPE}\"]}".equals(responseStr) - - webResource = client.resource( - appendToURL(registry_url, - "${RestPaths.REGISTRY_SERVICE}/${SliderKeys.APP_TYPE}")); - CuratorServiceInstances<ServiceInstanceData> services = webResource.type(MediaType.APPLICATION_JSON) - .get(CuratorServiceInstances.class); - assert services.services.size() == 1 - CuratorServiceInstance<ServiceInstanceData> service = services.services.get(0) - validateService(service) - - webResource = client.resource( - appendToURL(registry_url, - "${RestPaths.REGISTRY_SERVICE}/${SliderKeys.APP_TYPE}/"+id( - clustername))); - service = webResource.type(MediaType.APPLICATION_JSON) - .get(CuratorServiceInstance.class); - validateService(service) - - webResource = client.resource( - appendToURL( - registry_url, "${RestPaths.REGISTRY_ANYSERVICE}/${SliderKeys.APP_TYPE}")); - service = webResource.type(MediaType.APPLICATION_JSON) - .get(CuratorServiceInstance.class); - validateService(service) - - // some negative tests... - webResource = client.resource( - appendToURL(registry_url, "${RestPaths.REGISTRY_SERVICE}/dummy")) - services = webResource.type(MediaType.APPLICATION_JSON) - .get(CuratorServiceInstances.class); - assert services.services.size() == 0 - - try { - webResource = client.resource(appendToURL(registry_url, - "${RestPaths.REGISTRY_SERVICE}/${SliderKeys.APP_TYPE}/testregistryws99")); - - service = webResource.type(MediaType.APPLICATION_JSON) - .get(CuratorServiceInstance.class); - fail("should throw an exception for a 404 response....") - } catch (UniformInterfaceException e) { - assert e.response.getStatus() == 404 - } - - try { - webResource = client.resource( - appendToURL(registry_url, "${RestPaths.REGISTRY_ANYSERVICE}/dummy")); - - service = webResource.type(MediaType.APPLICATION_JSON) - .get(CuratorServiceInstance.class); - fail("should throw an exception for a 404 response....") - } catch (UniformInterfaceException e) { - assert e.response.getStatus() == 404 - } - } - - private void validateService(CuratorServiceInstance service) { - assert service.name.equals(SliderKeys.APP_TYPE) - assert service.serviceType == ServiceType.DYNAMIC - assert service.id.contains(CLUSTERNAME) - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestServiceInstanceSerDeser.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestServiceInstanceSerDeser.groovy b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestServiceInstanceSerDeser.groovy deleted file mode 100644 index 6a57cd9..0000000 --- a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestServiceInstanceSerDeser.groovy +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.registry.curator - -import groovy.util.logging.Slf4j -import org.apache.curator.x.discovery.ServiceInstance -import org.apache.curator.x.discovery.ServiceInstanceBuilder -import org.apache.curator.x.discovery.UriSpec -import org.apache.slider.core.persist.JsonSerDeser -import org.apache.slider.core.registry.info.ServiceInstanceData -import org.apache.slider.server.services.curator.CuratorServiceInstance -import org.junit.Test - -/** - * Here to identify why curator was complaining about deserializing - * its own published artifacts. - * - * ... its a Jackson version thing - */ -@Slf4j -class TestServiceInstanceSerDeser { - - JsonSerDeser<ServiceInstance> serializer = new JsonSerDeser<>( - ServiceInstance.class) - JsonSerDeser<CuratorServiceInstance> deser = new JsonSerDeser<>( - CuratorServiceInstance.class) - - @Test - public void testDefault() throws Throwable { - def builder = ServiceInstance.builder() - builder.name("defined") - buildAndRoundTrip("testDefault", builder) - } - - @Test - public void testEmpty() throws Throwable { - def builder = ServiceInstance.builder() - builder.address(null).id("").name("").port(0).uriSpec(null) - - buildAndRoundTrip("testEmpty", builder) - } - - @Test - public void testFilled() throws Throwable { - def builder = ServiceInstance.builder() - - builder.id("service").name("name").port(0) - .uriSpec(new UriSpec("http:{}:{}")) - - - buildAndRoundTrip("testFilled", builder) - } - - @Test - public void testPayload() throws Throwable { - def builder = ServiceInstance.builder() - builder.address(null).id("testPayload").name("").port(0).uriSpec(null) - - ServiceInstanceData data = new ServiceInstanceData() - data.serviceType = "testPayload" - data.externalView.documentsURL = "http://documents"; - builder.payload(data) - - def instance = buildAndRoundTrip("", builder) - - def payload = instance.payload as ServiceInstanceData - log.info("payload = $payload") - assert payload.externalView.documentsURL == "http://documents"; - } - - @Test - public void testHackedDeser() throws Throwable { - def builder = ServiceInstance.builder() - builder.address("localhost") - builder.id("service").name("name").port(8080) - .uriSpec(new UriSpec("http:{}:{}")) - .sslPort(443) - ServiceInstanceData data = new ServiceInstanceData() - data.externalView.documentsURL = "http://documents"; - - builder.payload(data) - log.info("Test: testHackedDeser") - String json = serialize(builder) - CuratorServiceInstance<ServiceInstanceData> curatorInstance = deser.fromJson(json) - log.info("resolved =$curatorInstance") - def payload = curatorInstance.payload - log.info("payload = $payload") - assert payload.externalView.documentsURL == "http://documents"; - - } - - def buildAndRoundTrip(String testname, ServiceInstanceBuilder builder) { - log.info("Test: $testname") - String json = serialize(builder) - - return serializer.fromJson(json) - } - - def String serialize(ServiceInstanceBuilder builder) { - ServiceInstance<ServiceInstanceData> instance = builder.build() - def json = serializer.toJson(instance) - log.info(json) - return json - } - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.groovy new file mode 100644 index 0000000..3883b25 --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.groovy @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.web.rest.registry + +import org.apache.hadoop.yarn.registry.client.binding.JsonSerDeser + +class PathEntryMarshalling + extends JsonSerDeser<PathEntryResource> { + public PathEntryMarshalling() { + super(PathEntryResource.class, new byte[0]); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.groovy new file mode 100644 index 0000000..44c8f05 --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.groovy @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.web.rest.registry + +import groovy.transform.CompileStatic +import org.junit.Test + +/** + * This test exists because Jersey appears to behave "oddly" + * when it comes to marshalling JSON, and some of the REST resources + * appear to have trouble. + * + * This test tries to isolate it + */ +@CompileStatic +class TestRegistryRestMarshalling { + + @Test + public void testDeser() throws Throwable { + PathEntryMarshalling pem = new PathEntryMarshalling(); + def unmarshalled = pem.fromResource( + "/org/apache/slider/server/appmaster/web/rest/registry/sample.json") + + def serviceRecord = unmarshalled.service + assert serviceRecord + assert serviceRecord.yarn_id !=null + assert serviceRecord.yarn_persistence != 0 + } + + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy new file mode 100644 index 0000000..877ba56 --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.web.rest.registry + +import com.sun.jersey.api.client.Client +import com.sun.jersey.api.client.ClientResponse +import com.sun.jersey.api.client.UniformInterfaceException +import com.sun.jersey.api.client.WebResource +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.hadoop.yarn.registry.client.api.RegistryConstants +import org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils +import org.apache.slider.api.StatusKeys +import org.apache.slider.client.SliderClient +import org.apache.slider.common.SliderKeys +import org.apache.slider.core.main.ServiceLauncher +import org.apache.slider.providers.agent.AgentTestBase +import org.apache.slider.server.appmaster.web.rest.RestPaths +import org.junit.Test + +import javax.ws.rs.core.MediaType + +import static org.apache.slider.common.params.Arguments.ARG_OPTION +import static org.apache.slider.core.registry.info.CustomRegistryConstants.* +import static org.apache.slider.providers.agent.AgentKeys.* +import static org.apache.slider.providers.agent.AgentTestUtils.createTestClient + +@CompileStatic +@Slf4j +class TestRegistryRestResources extends AgentTestBase { + + public static final String REGISTRY_URI = RestPaths.SLIDER_PATH_REGISTRY; + public static final String WADL = "vnd.sun.wadl+xml" + public static final String CLUSTERNAME = "test-registry-rest-resources" + + + @Test + public void testRestURIs() throws Throwable { + + def clustername = CLUSTERNAME + createMiniCluster( + clustername, + configuration, + 1, + 1, + 1, + true, + false) + Map<String, Integer> roles = [:] + File slider_core = new File(new File(".").absoluteFile, "src/test/python"); + File app_def_path = new File(app_def_pkg_path) + String agt_ver = "version" + File agt_ver_path = new File(slider_core, agt_ver) + String agt_conf = "agent.ini" + File agt_conf_path = new File(slider_core, agt_conf) + assert app_def_path.exists() + assert agt_ver_path.exists() + assert agt_conf_path.exists() + ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername, + roles, + [ + ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath, + ARG_OPTION, APP_DEF, toURIArg(app_def_path), + ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path), + ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path), + ], + true, true, + true) + SliderClient sliderClient = launcher.service + def report = waitForClusterLive(sliderClient) + def trackingUrl = report.trackingUrl + log.info("tracking URL is $trackingUrl") + def registry_url = appendToURL(trackingUrl, REGISTRY_URI) + + + def status = dumpClusterStatus(sliderClient, "agent AM") + def liveURL = status.getInfo(StatusKeys.INFO_AM_WEB_URL) + if (liveURL) { + registry_url = appendToURL(liveURL, REGISTRY_URI) + + } + + log.info("Registry is $registry_url") + log.info("stacks is ${liveURL}stacks") + log.info("conf is ${liveURL}conf") + + //WS get + Client client = createTestClient(); + + WebResource webResource = client.resource(registry_url); + + def jsonType = webResource.type(MediaType.APPLICATION_JSON) + PathEntryResource entryResource = jsonType + .get(PathEntryResource.class); + processResponse("/", jsonType.get(ClientResponse.class)) + assert entryResource.nodes != null; + assert entryResource.service == null; + + // test the available GET URIs + def userhome = RegistryOperationUtils.homePathForCurrentUser() + + def userServicesURL = appendToURL(registry_url, + userhome + RegistryConstants.PATH_USER_SERVICES) + webResource = client.resource(userServicesURL); + + + def jsonBuilder = webResource.type(MediaType.APPLICATION_JSON) + ClientResponse response = jsonBuilder.get(ClientResponse.class); + def responseStr = processResponse(userServicesURL, response) + + assert responseStr.contains("\"nodes\"") + assert responseStr.contains("/${SliderKeys.APP_TYPE}") + + entryResource = jsonBuilder.get(PathEntryResource.class) + assert entryResource.nodes.size() == 1; + assert entryResource.service == null; + + + def userServicesSlider = appendToURL(userServicesURL, SliderKeys.APP_TYPE) + webResource = client.resource( + userServicesSlider); + jsonBuilder = webResource.type(MediaType.APPLICATION_JSON); + response = jsonBuilder.get(ClientResponse.class); + processResponse(userServicesURL, response) + entryResource = jsonBuilder.get(PathEntryResource.class) + assert entryResource.nodes.size() == 1; + assert entryResource.service == null; + + def servicePath = entryResource.nodes[0] + + // now work with a real instances + + def instanceURL = appendToURL(userServicesSlider, clustername) + assert instanceURL.endsWith(servicePath) + + webResource = client.resource(instanceURL); + + // work with it via direct Jackson unmarshalling + responseStr = processResponse(instanceURL, webResource) + PathEntryMarshalling pem = new PathEntryMarshalling(); + def unmarshalled = pem.fromJson(responseStr) + def r1 = unmarshalled.service + assert r1 + assert r1.yarn_id != null + assert r1.yarn_persistence != 0 + + // and via the web resource AP + jsonBuilder = webResource.type(MediaType.APPLICATION_JSON); + entryResource = jsonBuilder.get(PathEntryResource.class); + + def serviceRecord = entryResource.service + assert serviceRecord != null; + assert serviceRecord.yarn_id != null; + def externalEndpoints = serviceRecord.external; + assert externalEndpoints.size() > 0 + + def am_ipc_protocol = AM_IPC_PROTOCOL + def epr = serviceRecord.getExternalEndpoint(am_ipc_protocol) + assert null != epr; + + assert null != serviceRecord.getExternalEndpoint(MANAGEMENT_REST_API) + assert null != serviceRecord.getExternalEndpoint(PUBLISHER_REST_API) + // internals + assert null != serviceRecord.getInternalEndpoint(AGENT_ONEWAY_REST_API) + assert null != serviceRecord.getInternalEndpoint(AGENT_SECURE_REST_API) + + // negative tests... + try { + webResource = client.resource( + appendToURL(registry_url, "/users/no-such-user")); + def clientResponse = webResource.get(ClientResponse.class) + assert 404 == clientResponse.status + + def body = processResponse(userServicesURL, webResource) + jsonBuilder = webResource.type(MediaType.APPLICATION_JSON); + entryResource = jsonBuilder.get(PathEntryResource.class); + + + fail("should throw an exception for a 404 response, got " + body) + } catch (UniformInterfaceException e) { + assert e.response.status == 404 + } + } + + public String processResponse(String asURL, WebResource response) { + def jsonBuilder = response.type(MediaType.APPLICATION_JSON) + def clientResponse = jsonBuilder.get(ClientResponse.class); + return processResponse(asURL, clientResponse) + } + + public String processResponse(String asURL, ClientResponse response) { + def responseStr = response.getEntity(String.class) + log.info(asURL + " ==>\n " + responseStr) + responseStr + } + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/slider-core/src/test/resources/log4j.properties b/slider-core/src/test/resources/log4j.properties index c1a524d..2fe1e49 100644 --- a/slider-core/src/test/resources/log4j.properties +++ b/slider-core/src/test/resources/log4j.properties @@ -29,8 +29,7 @@ log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n log4j.logger.org.apache.slider=DEBUG log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG - - +log4j.logger.org.apache.hadoop.yarn.registry=DEBUG #log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG #log4j.logger.org.apache.hadoop.yarn.service=DEBUG http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json b/slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json new file mode 100644 index 0000000..b2d59fd --- /dev/null +++ b/slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json @@ -0,0 +1,76 @@ +{ + "nodes": ["/users/example/services/org-apache-slider/test-registry-rest-resources/components"], "service": { + "registrationTime": 0, + "description": "Slider Application Master", + "yarn_id": "application_1411664296263_0001", + "yarn_persistence": 1, + "external": [ + { + "api": "org.apache.slider.appmaster", + "addressType": "host/port", + "protocolType": "hadoop/protobuf", + "addresses": [ + ["server", "63385"] + ] + }, + { + "api": "org.apache.http.UI", + "addressType": "uri", + "protocolType": "webui", + "addresses": [ + ["http://server:63391";] + ] + }, + { + "api": "org.apache.slider.management", + "addressType": "uri", + "protocolType": "REST", + "addresses": [ + ["http://server:63391/ws/v1/slider/mgmt";] + ] + }, + { + "api": "org.apache.slider.publisher", + "addressType": "uri", + "protocolType": "REST", + "addresses": [ + ["http://server:63391/ws/v1/slider/publisher";] + ] + }, + { + "api": "org.apache.slider.registry", + "addressType": "uri", + "protocolType": "REST", + "addresses": [ + ["http://server:63391/ws/v1/slider/registry";] + ] + }, + { + "api": "org.apache.slider.publisher.configurations", + "addressType": "uri", + "protocolType": "REST", + "addresses": [ + ["http://server:63391/ws/v1/slider/publisher/slider";] + ] + } + ], + "internal": [ + { + "api": "org.apache.slider.agents.secure", + "addressType": "uri", + "protocolType": "REST", + "addresses": [ + ["https://server:63390/ws/v1/slider/agents";] + ] + }, + { + "api": "org.apache.slider.agents.oneway", + "addressType": "uri", + "protocolType": "REST", + "addresses": [ + ["https://server:63389/ws/v1/slider/agents";] + ] + } + ] +} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/FunctionalHBaseClusterIT.groovy ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/FunctionalHBaseClusterIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/FunctionalHBaseClusterIT.groovy index 84e55f7..5137101 100644 --- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/FunctionalHBaseClusterIT.groovy +++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/FunctionalHBaseClusterIT.groovy @@ -30,7 +30,7 @@ import org.apache.slider.common.SliderKeys import org.apache.slider.common.SliderXmlConfKeys import org.apache.slider.common.params.Arguments import org.apache.slider.common.tools.ConfigHelper -import org.apache.slider.core.registry.info.RegistryNaming + import org.apache.slider.funtest.framework.FuntestProperties import org.apache.slider.providers.hbase.HBaseConfigFileOptions import org.apache.slider.providers.hbase.HBaseTestUtils @@ -184,10 +184,7 @@ public class FunctionalHBaseClusterIT extends HBaseCommandTestBase [ARG_LIST, ARG_NAME, "cluster-with-no-name"]) // how to work out the current service name? - def name = RegistryNaming.createRegistryName(clustername, - System.getProperty("user.name"), - SliderKeys.APP_TYPE, - 1) + def name = clustername registry([ARG_LIST, ARG_VERBOSE, ARG_NAME, name]) registry([ARG_LISTCONF, ARG_NAME, name]) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy index 9c53d2c..422ee8d 100644 --- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy +++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy @@ -93,7 +93,7 @@ class TestHBaseMaster extends HBaseMiniClusterTestBase { assert hbaseInstances.size() == 1 ServiceRecord hbaseServiceData = hbaseInstances[0] log.info "HBase service 0 == $hbaseServiceData" - assert hbaseServiceData.id + assert hbaseServiceData.yarn_id RegistryRetriever retriever = new RegistryRetriever(hbaseServiceData) log.info retriever.toString()
