SLIDER-719: bonding jersey to UrlConnectionOperations
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/61f4743f Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/61f4743f Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/61f4743f Branch: refs/heads/develop Commit: 61f4743f83d73e255eee5744be3e72bbb9c60754 Parents: 901f38c Author: Steve Loughran <ste...@apache.org> Authored: Fri Jan 16 16:57:09 2015 +0000 Committer: Steve Loughran <ste...@apache.org> Committed: Tue Jan 20 14:50:23 2015 +0000 ---------------------------------------------------------------------- pom.xml | 5 +- .../core/restclient/UgiJerseyBinding.java | 79 ++++++++++++++++++++ .../restclient/UrlConnectionOperations.java | 2 +- .../slider/agent/rest/RestTestDelegates.groovy | 20 ++++- .../slider/agent/rest/TestStandaloneREST.groovy | 7 +- .../agent/TestAgentAMManagementWS.groovy | 4 +- .../publisher/TestPublisherRestResources.groovy | 2 +- .../apache/slider/test/SliderTestUtils.groovy | 61 ++++++++++++--- .../funtest/lifecycle/AgentWebPagesIT.groovy | 3 +- 9 files changed, 159 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index cf989e4..b206d66 100644 --- a/pom.xml +++ b/pom.xml @@ -154,6 +154,7 @@ <gson.version>2.2.2</gson.version> <guice.version>3.0</guice.version> <httpclient.version>3.1</httpclient.version> + <httpclient4.version>4.2.5</httpclient4.version> <jackson.version>1.9.13</jackson.version> <jcommander.version>1.30</jcommander.version> @@ -645,13 +646,13 @@ <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> - <version>4.2.5</version> + <version>${httpclient4.version}</version> </dependency> <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpcore</artifactId> - <version>4.2.5</version> + <version>${httpclient4.version}</version> </dependency> <!-- ======================================================== --> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java b/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java new file mode 100644 index 0000000..6af8e23 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.restclient; + +import com.google.common.base.Preconditions; +import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; +import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; +import org.apache.hadoop.security.authentication.client.AuthenticationException; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; + +/** + * Class to bond to a Jersey client, for UGI integration and SPNEGO. + * <p> + * Usage: create an instance, then when creating a Jersey <code>Client</code> + * pass in to the constructor the handler provided by {@link #getHandler()} + * + * @see https://jersey.java.net/apidocs/1.17/jersey/com/sun/jersey/client/urlconnection/HttpURLConnectionFactory.html + */ +public class UgiJerseyBinding implements + HttpURLConnectionFactory { + private final UrlConnectionOperations operations; + private final URLConnectionClientHandler handler; + + /** + * Construct an instance + * @param operations operations instance + */ + @SuppressWarnings("ThisEscapedInObjectConstruction") + public UgiJerseyBinding(UrlConnectionOperations operations) { + Preconditions.checkArgument(operations != null, "Null operations"); + this.operations = operations; + handler = new URLConnectionClientHandler(this); + } + + /** + * Get a URL connection. + * @param url + * @return the connection + * @throws IOException any problem. {@link AuthenticationException} + * errors are wrapped + */ + @Override + public HttpURLConnection getHttpURLConnection(URL url) throws IOException { + try { + return operations.openConnection(url); + } catch (AuthenticationException e) { + throw new IOException(e); + } + } + + public UrlConnectionOperations getOperations() { + return operations; + } + + public URLConnectionClientHandler getHandler() { + return handler; + } +} + + http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java b/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java index fa0da5a..c51914a 100644 --- a/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java +++ b/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java @@ -40,7 +40,7 @@ import java.net.URL; * Operations on the JDK UrlConnection class. This uses WebHDFS * methods to set up the operations. */ -public class UrlConnectionOperations extends Configured { +public class UrlConnectionOperations extends Configured { private static final Logger log = LoggerFactory.getLogger(UrlConnectionOperations.class); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy index 3431175..0826204 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy @@ -18,6 +18,8 @@ package org.apache.slider.agent.rest +import com.sun.jersey.api.client.Client +import com.sun.jersey.api.client.WebResource import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.hadoop.yarn.webapp.NotFoundException @@ -27,9 +29,11 @@ import org.apache.slider.api.types.SerializedContainerInformation import org.apache.slider.core.conf.AggregateConf import org.apache.slider.core.conf.ConfTree import org.apache.slider.core.conf.ConfTreeOperations +import org.apache.slider.core.registry.docstore.PublishedConfiguration import org.apache.slider.core.restclient.HttpOperationResponse import org.apache.slider.core.restclient.HttpVerb import org.apache.slider.core.restclient.UrlConnectionOperations +import org.apache.slider.server.appmaster.web.rest.RestPaths import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource import org.apache.slider.server.appmaster.web.rest.application.resources.PingResource import org.apache.slider.test.Outcome @@ -53,11 +57,14 @@ class RestTestDelegates extends SliderTestUtils { public static final String TEST_GLOBAL_OPTION_PRESENT = "present" final String appmaster; + final String application; RestTestDelegates(String appmaster) { this.appmaster = appmaster + application = appendToURL(appmaster, RestPaths.SLIDER_PATH_APPLICATION) } + public void testCodahaleOperations() throws Throwable { describe "Codahale operations" getWebPage(appmaster) @@ -89,6 +96,15 @@ class RestTestDelegates extends SliderTestUtils { assert 0 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_COMPLETED) assert 0 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_RELEASING) } + + + public void testRestletOperations() throws Throwable { + Client client = createJerseyClient() + String path = appendToURL(application, LIVE_RESOURCES) + WebResource webResource = client.resource(path) + webResource.type(MediaType.APPLICATION_JSON) + .get(ConfTree.class); + } public void testLiveContainers() throws Throwable { describe "Application REST ${LIVE_CONTAINERS}" @@ -211,7 +227,7 @@ class RestTestDelegates extends SliderTestUtils { HttpVerb verb, URL pingUrl, String payload) { - return pingAction(connectionFactory, verb, pingUrl, payload) + return pingAction(connectionOperations, verb, pingUrl, payload) } private HttpOperationResponse pingAction( @@ -244,7 +260,7 @@ class RestTestDelegates extends SliderTestUtils { String target = appendToURL(appmaster, SLIDER_PATH_APPLICATION, ACTION_STOP) describe "Stop URL $target" URL targetUrl = new URL(target) - def outcome = connectionFactory.execHttpOperation( + def outcome = connectionOperations.execHttpOperation( HttpVerb.POST, targetUrl, new byte[0], http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy index a3378a8..22bc4ae 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy @@ -62,14 +62,14 @@ class TestStandaloneREST extends AgentMiniClusterTestBase { def realappmaster = report.originalTrackingUrl // set up url config to match - initConnectionFactory(launcher.configuration) + initHttpTestSupport(launcher.configuration) - execHttpRequest(WEB_STARTUP_TIME) { + execOperation(WEB_STARTUP_TIME) { GET(realappmaster) } - execHttpRequest(WEB_STARTUP_TIME) { + execOperation(WEB_STARTUP_TIME) { def metrics = GET(realappmaster, SYSTEM_METRICS) log.info metrics } @@ -90,6 +90,7 @@ class TestStandaloneREST extends AgentMiniClusterTestBase { RestTestDelegates proxied = new RestTestDelegates(appmaster) RestTestDelegates direct = new RestTestDelegates(realappmaster) + direct.testRestletOperations(); proxied.testCodahaleOperations() direct.testCodahaleOperations() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy index 2e2e62b..7eb1d17 100644 --- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy @@ -153,7 +153,7 @@ class TestAgentAMManagementWS extends AgentTestBase { log.info("tracking URL is $proxyAM") // spin awaiting the agent web page coming up. - execHttpRequest(WEB_STARTUP_TIME) { + execOperation(WEB_STARTUP_TIME) { GET(proxyAM) } @@ -173,7 +173,7 @@ class TestAgentAMManagementWS extends AgentTestBase { log.info "AM live, now fetching agent at $agent_url" // spin awaiting the agent web page coming up. - execHttpRequest(WEB_STARTUP_TIME) { + execOperation(WEB_STARTUP_TIME) { GET(agent_url) } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy index e3f38eb..29f0510 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy @@ -107,7 +107,7 @@ class TestPublisherRestResources extends AgentTestBase { webResource = client.resource(sliderConfigset + "dummy-site"); - execHttpRequest(30000) { + execOperation(30000) { GET(sliderConfigset) } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy index 2c6b5fe..9dd2828 100644 --- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy @@ -18,6 +18,10 @@ package org.apache.slider.test +import com.sun.jersey.api.client.Client +import com.sun.jersey.api.client.config.ClientConfig +import com.sun.jersey.api.client.config.DefaultClientConfig +import com.sun.jersey.api.json.JSONConfiguration import groovy.json.JsonOutput import groovy.transform.CompileStatic import groovy.util.logging.Slf4j @@ -55,6 +59,7 @@ import org.apache.slider.core.main.ServiceLaunchException import org.apache.slider.core.main.ServiceLauncher import org.apache.slider.core.persist.JsonSerDeser import org.apache.slider.core.registry.docstore.PublishedConfigSet +import org.apache.slider.core.restclient.UgiJerseyBinding import org.apache.slider.core.restclient.UrlConnectionOperations import org.apache.slider.server.appmaster.web.HttpCacheHeaders import org.apache.slider.server.appmaster.web.rest.RestPaths @@ -450,7 +455,8 @@ class SliderTestUtils extends Assert { } /** - * Fetch a web page + * Fetch a web page using HttpClient 3.1. + * This <i>DOES NOT</i> work with secure connections. * @param url URL * @return the response body */ @@ -478,6 +484,9 @@ class SliderTestUtils extends Assert { /** * Fetches a web page asserting that the response code is between 200 and 400. + * This <i>DOES NOT</i> work with secure connections. + * <p> + * * Will error on 400 and 500 series response codes and let 200 and 300 through. * @param url URL to get as string * @return body of response @@ -548,7 +557,7 @@ class SliderTestUtils extends Assert { /** * Fetches a web page asserting that the response code is between 200 and 400. * Will error on 400 and 500 series response codes and let 200 and 300 through. - * + * <p> * if security is enabled, this uses SPNEGO to auth * @param page * @return body of response @@ -559,13 +568,14 @@ class SliderTestUtils extends Assert { } /** - * Execute any of the http requests, swallowing exceptions until - * eventually they time out + * Execute any operation provided as a closure which returns a string, swallowing exceptions until + * eventually they time out. + * * @param timeout * @param operation * @return */ - public static String execHttpRequest(int timeout, Closure operation) { + public static String execOperation(int timeout, Closure operation) { Duration duration = new Duration(timeout).start() Exception ex = new IOException("limit exceeded before starting"); while (!duration.limitExceeded) { @@ -581,12 +591,41 @@ class SliderTestUtils extends Assert { throw ex; } - static UrlConnectionOperations connectionFactory + /** + * Static factory for URL connections + */ + static UrlConnectionOperations connectionOperations + static UgiJerseyBinding jerseyBinding; - public static def initConnectionFactory(Configuration conf) { - connectionFactory = new UrlConnectionOperations(conf); + + /** + * Static initializer of the connection operations + * @param conf config + */ + public static synchronized void initHttpTestSupport(Configuration conf) { + connectionOperations = new UrlConnectionOperations(conf); + jerseyBinding = new UgiJerseyBinding(connectionOperations) + } + /** + * Check for the HTTP support being initialized + */ + public static synchronized void assertHttpSupportInitialized() { + assert connectionOperations + assert jerseyBinding + } + + /** + * Create Jersey client + * @return + */ + public static Client createJerseyClient() { + assertHttpSupportInitialized() + ClientConfig clientConfig = new DefaultClientConfig(); + clientConfig.features[JSONConfiguration.FEATURE_POJO_MAPPING] =Boolean.TRUE; + return new Client(jerseyBinding.handler, clientConfig); + } /** * Fetches a web page asserting that the response code is between 200 and 400. @@ -594,7 +633,7 @@ class SliderTestUtils extends Assert { * * if security is enabled, this uses SPNEGO to auth * <p> - * Relies on {@link #initConnectionFactory(org.apache.hadoop.conf.Configuration)} + * Relies on {@link #initHttpTestSupport(org.apache.hadoop.conf.Configuration)} * to have been called. * * @param path path to page @@ -603,11 +642,11 @@ class SliderTestUtils extends Assert { */ public static String getWebPage(String path, Closure connectionChecks = null) { assert path - assert null != connectionFactory + assertHttpSupportInitialized() log.info("Fetching HTTP content at " + path); URL url = new URL(path) - def outcome = connectionFactory.execGet(url) + def outcome = connectionOperations.execGet(url) String body = new String(outcome.data) return body; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy index ce1d955..de8dcdf 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy @@ -29,7 +29,6 @@ import org.apache.slider.common.tools.ConfigHelper import org.apache.slider.funtest.framework.AgentCommandTestBase import org.apache.slider.funtest.framework.FuntestProperties import org.apache.slider.funtest.framework.SliderShell -import org.apache.slider.server.appmaster.web.rest.RestPaths import org.junit.After import org.junit.Before import org.junit.Test @@ -82,7 +81,7 @@ public class AgentWebPagesIT extends AgentCommandTestBase def conf = SLIDER_CONFIG - initConnectionFactory(conf) + initHttpTestSupport(conf) def appId = ensureYarnApplicationIsUp(launchReportFile) assert appId