This is an automated email from the ASF dual-hosted git repository. eyang pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push: new d81d45f YARN-9956. Improved connection error message for YARN ApiServerClient. Contributed by Prabhu Joseph d81d45f is described below commit d81d45ff2fc9a1c424222e021f9306bf64c916b2 Author: Eric Yang <ey...@apache.org> AuthorDate: Mon Jan 6 13:24:16 2020 -0500 YARN-9956. Improved connection error message for YARN ApiServerClient. Contributed by Prabhu Joseph --- .../yarn/service/client/ApiServiceClient.java | 78 ++++++++++++---------- .../yarn/service/client/TestApiServiceClient.java | 22 ++++++ .../service/client/TestSecureApiServiceClient.java | 2 + .../hadoop/yarn/client/util/YarnClientUtils.java | 2 +- 4 files changed, 68 insertions(+), 36 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java index 834bb03..3c2c3c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.client.api.AppAdminClient; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.util.YarnClientUtils; +import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.service.api.records.Component; @@ -94,7 +95,7 @@ public class ApiServiceClient extends AppAdminClient { /** * Calculate Resource Manager address base on working REST API. */ - String getRMWebAddress() { + String getRMWebAddress() throws IOException { Configuration conf = getConfig(); String scheme = "http://"; String path = "/app/v1/services/version"; @@ -105,43 +106,50 @@ public class ApiServiceClient extends AppAdminClient { rmAddress = conf .get("yarn.resourcemanager.webapp.https.address"); } - boolean useKerberos = UserGroupInformation.isSecurityEnabled(); - List<String> rmServers = getRMHAWebAddresses(conf); - for (String host : rmServers) { - try { - Client client = Client.create(); - client.setFollowRedirects(false); - StringBuilder sb = new StringBuilder(); - sb.append(scheme) - .append(host) - .append(path); - if (!useKerberos) { - try { - String username = UserGroupInformation.getCurrentUser().getShortUserName(); - sb.append("?user.name=") - .append(username); - } catch (IOException e) { - LOG.debug("Fail to resolve username: {}", e); + + if (HAUtil.isHAEnabled(conf)) { + boolean useKerberos = UserGroupInformation.isSecurityEnabled(); + List<String> rmServers = getRMHAWebAddresses(conf); + StringBuilder diagnosticsMsg = new StringBuilder(); + for (String host : rmServers) { + try { + Client client = Client.create(); + client.setFollowRedirects(false); + StringBuilder sb = new StringBuilder(); + sb.append(scheme) + .append(host) + .append(path); + if (!useKerberos) { + try { + String username = UserGroupInformation.getCurrentUser() + .getShortUserName(); + sb.append("?user.name=") + .append(username); + } catch (IOException e) { + LOG.debug("Fail to resolve username: {}", e); + } } + Builder builder = client + .resource(sb.toString()).type(MediaType.APPLICATION_JSON); + if (useKerberos) { + String[] server = host.split(":"); + String challenge = YarnClientUtils.generateToken(server[0]); + builder.header(HttpHeaders.AUTHORIZATION, "Negotiate " + + challenge); + LOG.debug("Authorization: Negotiate {}", challenge); + } + ClientResponse test = builder.get(ClientResponse.class); + if (test.getStatus() == 200) { + return scheme + host; + } + } catch (Exception e) { + LOG.info("Fail to connect to: " + host); + LOG.debug("Root cause: ", e); + diagnosticsMsg.append("Error connecting to " + host + + " due to " + e.getMessage() + "\n"); } - Builder builder = client - .resource(sb.toString()).type(MediaType.APPLICATION_JSON); - if (useKerberos) { - String[] server = host.split(":"); - String challenge = YarnClientUtils.generateToken(server[0]); - builder.header(HttpHeaders.AUTHORIZATION, "Negotiate " + - challenge); - LOG.debug("Authorization: Negotiate {}", challenge); - } - ClientResponse test = builder.get(ClientResponse.class); - if (test.getStatus() == 200) { - rmAddress = host; - break; - } - } catch (Exception e) { - LOG.info("Fail to connect to: "+host); - LOG.debug("Root cause: {}", e); } + throw new IOException(diagnosticsMsg.toString()); } return scheme+rmAddress; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java index afdf16e..65849a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java @@ -29,6 +29,7 @@ import javax.servlet.http.HttpServletResponse; import com.google.common.collect.Lists; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; @@ -118,6 +119,27 @@ public class TestApiServiceClient { } @Test + public void testGetRMWebAddress() throws Exception { + Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); + conf.set(YarnConfiguration.RM_HA_IDS, "rm1"); + conf.set(YarnConfiguration.RM_HA_ID, "rm1"); + conf.set("yarn.resourcemanager.webapp.address.rm1", "localhost:0"); + ApiServiceClient asc1 = new ApiServiceClient(conf); + boolean exceptionCaught = false; + String diagnosticsMsg = null; + try { + String rmWebAddress = asc1.getRMWebAddress(); + } catch (IOException e){ + exceptionCaught = true; + diagnosticsMsg = e.getMessage(); + } + assertTrue("ApiServiceClient failed to throw exception", exceptionCaught); + assertTrue("Exception Message does not match", + diagnosticsMsg.contains("Error connecting to localhost:0")); + } + + @Test public void testLaunch() { String fileName = "target/test-classes/example-app.json"; String appName = "example-app"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java index 8c7da62..e2d613a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java @@ -41,6 +41,7 @@ import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.SaslRpcServer.QualityOfProtection; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.client.util.YarnClientUtils; import org.apache.log4j.Logger; import org.eclipse.jetty.server.Server; @@ -152,6 +153,7 @@ public class TestSecureApiServiceClient extends KerberosSecurityTestcase { rmServers.add("localhost:8088"); testConf.set("yarn.resourcemanager.webapp.address", "localhost:8088"); + testConf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); asc = new ApiServiceClient() { @Override List<String> getRMHAWebAddresses(Configuration conf) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java index abed6c6..94b13a0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java @@ -247,7 +247,7 @@ public abstract class YarnClientUtils { StandardCharsets.US_ASCII); } catch (GSSException | IllegalAccessException | NoSuchFieldException | ClassNotFoundException e) { - LOG.error("Error: {}", e); + LOG.error("Error: ", e); throw new AuthenticationException(e); } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org