Repository: incubator-slider Updated Branches: refs/heads/develop 87ed02cab -> 1742ab232
SLIDER-907 AgentWebPagesIT#testAgentWeb fails on wire encrypted clusters: publish proxied values from outset Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1742ab23 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1742ab23 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1742ab23 Branch: refs/heads/develop Commit: 1742ab232ba65e95cb287b4f040def194effbf1f Parents: 87ed02c Author: Steve Loughran <ste...@apache.org> Authored: Fri Dec 11 15:03:21 2015 +0000 Committer: Steve Loughran <ste...@apache.org> Committed: Fri Dec 11 15:03:21 2015 +0000 ---------------------------------------------------------------------- .../slider/providers/ProviderService.java | 8 +++--- .../slideram/SliderAMProviderService.java | 6 ++--- .../server/appmaster/SliderAppMaster.java | 21 +++++++++++++-- .../server/appmaster/web/rest/RestPaths.java | 14 ++++++++-- .../TestStandaloneYarnRegistryAM.groovy | 28 ++++++++++++-------- 5 files changed, 55 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1742ab23/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java index 28cef01..1a18db6 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java @@ -186,10 +186,10 @@ public interface ProviderService extends ProviderCore, /** * Prior to going live -register the initial service registry data - * @param amWebURI - * @param agentOpsURI - * @param agentStatusURI - * @param serviceRecord + * @param amWebURI URL to the AM. This may be proxied, so use relative paths + * @param agentOpsURI URI for agent operations. This will not be proxied + * @param agentStatusURI URI For agent status. Again: no proxy + * @param serviceRecord service record to build up */ void applyInitialRegistryDefinitions(URL amWebURI, URL agentOpsURI, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1742ab23/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java index e382058..f0dab8c 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java @@ -141,10 +141,10 @@ public class SliderAMProviderService extends AbstractProviderService implements try { - URL managementAPI = new URL(amWebURI, SLIDER_PATH_MANAGEMENT); - URL registryREST = new URL(amWebURI, SLIDER_PATH_REGISTRY ); + URL managementAPI = new URL(amWebURI, RELATIVE_PATH_MANAGEMENT); + URL registryREST = new URL(amWebURI, RELATIVE_PATH_REGISTRY); - URL publisherURL = new URL(amWebURI, SLIDER_PATH_PUBLISHER); + URL publisherURL = new URL(amWebURI, RELATIVE_PATH_PUBLISHER); // Set the configurations URL. http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1742ab23/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index ef5290b..415a597 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -281,6 +282,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized") private String appMasterTrackingUrl = ""; + /** Proxied app master URL (as retrieved from AM report at launch time) */ + @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized") + private String appMasterProxiedUrl = ""; + /** Application Attempt Id ( combination of attemptId and fail count )*/ private ApplicationAttemptId appAttemptID; @@ -823,6 +828,17 @@ public class SliderAppMaster extends AbstractSliderLaunchedService } } + // look up the application itself -this is needed to get the proxied + // URL of the AM, for registering endpoints. + // this call must be made after the AM has registered itself, obviously + ApplicationAttemptReport report = yarnClient.getApplicationAttemptReport( + appAttemptID); + appMasterProxiedUrl = report.getTrackingUrl(); + if (SliderUtils.isUnset(appMasterProxiedUrl)) { + log.warn("Proxied URL is not set in application report"); + appMasterProxiedUrl = appMasterTrackingUrl; + } + // extract container list liveContainers = amRegistrationData.getContainersFromPreviousAttempts(); @@ -949,7 +965,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService // log the YARN and web UIs log.info("RM Webapp address {}", serviceConf.get(YarnConfiguration.RM_WEBAPP_ADDRESS)); - log.info("slider Webapp address {}", appMasterTrackingUrl); + log.info("Slider webapp address {} proxied at {}", + appMasterTrackingUrl, appMasterProxiedUrl); // Start the Slider AM provider sliderAMProvider.start(); @@ -1190,7 +1207,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService // the registry is running, so register services - URL amWebURI = new URL(appMasterTrackingUrl); + URL amWebURI = new URL(appMasterProxiedUrl); URL agentOpsURI = new URL(agentOpsUrl); URL agentStatusURI = new URL(agentStatusUrl); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1742ab23/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java index b90eb62..ae9eb0f 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java @@ -35,8 +35,10 @@ public class RestPaths { * agent content root: {@value} */ public static final String WS_AGENT_CONTEXT_ROOT = "/" + AGENT_WS_CONTEXT; - public static final String SLIDER_CONTEXT_ROOT = WS_CONTEXT_ROOT +"/v1/slider"; - public static final String SLIDER_AGENT_CONTEXT_ROOT = WS_AGENT_CONTEXT_ROOT +"/v1/slider"; + public static final String V1_SLIDER = "/v1/slider"; + public static final String SLIDER_CONTEXT_ROOT = WS_CONTEXT_ROOT + V1_SLIDER; + public static final String RELATIVE_API = WS_CONTEXT + V1_SLIDER; + public static final String SLIDER_AGENT_CONTEXT_ROOT = WS_AGENT_CONTEXT_ROOT + V1_SLIDER; public static final String MANAGEMENT = "mgmt"; public static final String SLIDER_SUBPATH_MANAGEMENT = "/" + MANAGEMENT; public static final String SLIDER_SUBPATH_AGENTS = "/agents"; @@ -49,6 +51,9 @@ public class RestPaths { public static final String SLIDER_PATH_MANAGEMENT = SLIDER_CONTEXT_ROOT + SLIDER_SUBPATH_MANAGEMENT; + public static final String RELATIVE_PATH_MANAGEMENT = RELATIVE_API + + SLIDER_SUBPATH_MANAGEMENT; + /** * Agents: {@value} */ @@ -61,6 +66,9 @@ public class RestPaths { public static final String SLIDER_PATH_PUBLISHER = SLIDER_CONTEXT_ROOT + SLIDER_SUBPATH_PUBLISHER; + public static final String RELATIVE_PATH_PUBLISHER = RELATIVE_API + + SLIDER_SUBPATH_PUBLISHER; + /** * Registry subpath: {@value} */ @@ -71,6 +79,8 @@ public class RestPaths { */ public static final String SLIDER_PATH_REGISTRY = SLIDER_CONTEXT_ROOT + SLIDER_SUBPATH_REGISTRY; + public static final String RELATIVE_PATH_REGISTRY = RELATIVE_API + + SLIDER_SUBPATH_REGISTRY; /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1742ab23/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy index 139860a..56fb5f2 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy @@ -18,10 +18,13 @@ package org.apache.slider.agent.standalone +import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.hadoop.fs.FileUtil import org.apache.hadoop.fs.PathNotFoundException import org.apache.hadoop.registry.client.binding.RegistryPathUtils +import org.apache.hadoop.registry.client.types.AddressTypes +import org.apache.hadoop.registry.client.types.ProtocolTypes import org.apache.hadoop.yarn.api.records.ApplicationReport import org.apache.hadoop.yarn.api.records.YarnApplicationState import org.apache.hadoop.yarn.conf.YarnConfiguration @@ -31,7 +34,6 @@ import org.apache.hadoop.registry.client.impl.RegistryOperationsClient import org.apache.hadoop.registry.client.types.RegistryPathStatus import org.apache.hadoop.registry.client.types.ServiceRecord import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes -import org.apache.slider.common.SliderExitCodes import org.apache.slider.common.params.ActionResolveArgs import org.apache.slider.common.params.Arguments import org.apache.slider.core.exceptions.NotFoundException @@ -59,7 +61,7 @@ import static org.apache.slider.core.registry.info.CustomRegistryConstants.* /** * work with a YARN registry */ -//@CompileStatic +@CompileStatic @Slf4j class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase { @@ -165,15 +167,19 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase { assert serviceRecord[YarnRegistryAttributes.YARN_PERSISTENCE] != "" def externalEndpoints = serviceRecord.external; assert externalEndpoints.size() > 0 -/* - - def am_ipc_protocol = AM_IPC_PROTOCOL - serviceRecord.getExternalEndpoint(am_ipc_protocol) - assert null != am_ipc_protocol; -*/ assert null != serviceRecord.getExternalEndpoint(MANAGEMENT_REST_API) assert null != serviceRecord.getExternalEndpoint(PUBLISHER_REST_API) + def publisherEPR = serviceRecord.getExternalEndpoint(PUBLISHER_REST_API) + assert publisherEPR.protocolType == ProtocolTypes.PROTOCOL_REST + assert publisherEPR.addressType == AddressTypes.ADDRESS_URI + def addr = publisherEPR.addresses[0][AddressTypes.ADDRESS_URI] + def proxiedURL = new URL(addr) + def appId = report.applicationId.toString() + // something like http://host:49257/proxy/application_1449844996855_0001/ws/v1/slider/publisher + String publisherPath = "/proxy/$appId/" + RestPaths.RELATIVE_PATH_PUBLISHER + assert proxiedURL.path == publisherPath + // internals assert null != serviceRecord.getInternalEndpoint(AGENT_ONEWAY_REST_API) assert null != serviceRecord.getInternalEndpoint(AGENT_SECURE_REST_API) @@ -323,8 +329,8 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase { String confJSON = GET(yarnSitePublisher) // log.info(confJSON) - JsonSerDeser< PublishedConfiguration> confSerDeser = - new JsonSerDeser<PublishedConfiguration>(PublishedConfiguration) + JsonSerDeser<PublishedConfiguration> confSerDeser = + new JsonSerDeser<>(PublishedConfiguration) publishedYarnSite = confSerDeser.fromJson(confJSON) @@ -488,7 +494,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase { assert LauncherExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs) //look for a different user - assertFailsWithException(SliderExitCodes.EXIT_NOT_FOUND, "") { + assertFailsWithException(LauncherExitCodes.EXIT_NOT_FOUND, "") { def args = new ActionRegistryArgs( name: clustername, user: "unknown",