YARN-4746. yarn web services should convert parse failures of appId, appAttemptId and containerId to 400. Contributed by Bibin A Chundatt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5092c941 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5092c941 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5092c941 Branch: refs/heads/YARN-3368 Commit: 5092c94195a63bd2c3e36d5a74b4c061cea1b847 Parents: da614ca Author: naganarasimha <naganarasimha...@apache.com> Authored: Mon Apr 4 16:25:03 2016 +0530 Committer: naganarasimha <naganarasimha...@apache.com> Committed: Mon Apr 4 16:25:03 2016 +0530 ---------------------------------------------------------------------- .../apache/hadoop/yarn/util/ConverterUtils.java | 16 ++++++++-- .../hadoop/yarn/webapp/util/WebAppUtils.java | 22 ++++++++++++++ .../hadoop/yarn/server/webapp/WebServices.java | 22 +++++++++++--- .../nodemanager/webapp/NMWebServices.java | 6 ++-- .../webapp/TestNMWebServicesApps.java | 9 ++++-- .../resourcemanager/webapp/RMWebServices.java | 32 ++------------------ .../webapp/TestRMWebServicesApps.java | 24 +++++++++------ .../TestRMWebServicesAppsModification.java | 10 ++++-- 8 files changed, 87 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5092c941/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java index e9674cf..acd29fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java @@ -122,8 +122,20 @@ public class ConverterUtils { public static ApplicationId toApplicationId(RecordFactory recordFactory, String appIdStr) { Iterator<String> it = _split(appIdStr).iterator(); - it.next(); // prefix. TODO: Validate application prefix - return toApplicationId(recordFactory, it); + if (!it.next().equals(APPLICATION_PREFIX)) { + throw new IllegalArgumentException("Invalid ApplicationId prefix: " + + appIdStr + ". The valid ApplicationId should start with prefix " + + APPLICATION_PREFIX); + } + try { + return toApplicationId(recordFactory, it); + } catch (NumberFormatException n) { + throw new IllegalArgumentException("Invalid ApplicationId: " + appIdStr, + n); + } catch (NoSuchElementException e) { + throw new IllegalArgumentException("Invalid ApplicationId: " + appIdStr, + e); + } } private static ApplicationId toApplicationId(RecordFactory recordFactory, http://git-wip-us.apache.org/repos/asf/hadoop/blob/5092c941/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index f8e67ee..faf4a77 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -33,9 +33,14 @@ import org.apache.hadoop.http.HttpConfig.Policy; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.RMHAUtils; +import org.apache.hadoop.yarn.webapp.BadRequestException; +import org.apache.hadoop.yarn.webapp.NotFoundException; @Private @Evolving @@ -378,4 +383,21 @@ public class WebAppUtils { } return password; } + + public static ApplicationId parseApplicationId(RecordFactory recordFactory, + String appId) { + if (appId == null || appId.isEmpty()) { + throw new NotFoundException("appId, " + appId + ", is empty or null"); + } + ApplicationId aid = null; + try { + aid = ConverterUtils.toApplicationId(recordFactory, appId); + } catch (Exception e) { + throw new BadRequestException(e); + } + if (aid == null) { + throw new NotFoundException("app with id " + appId + " not found"); + } + return aid; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5092c941/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java index 40e40c9..19ea301 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java @@ -429,7 +429,12 @@ public class WebServices { if (appId == null || appId.isEmpty()) { throw new NotFoundException("appId, " + appId + ", is empty or null"); } - ApplicationId aid = ConverterUtils.toApplicationId(appId); + ApplicationId aid = null; + try { + aid = ConverterUtils.toApplicationId(appId); + } catch (Exception e) { + throw new BadRequestException(e); + } if (aid == null) { throw new NotFoundException("appId is null"); } @@ -442,8 +447,12 @@ public class WebServices { throw new NotFoundException("appAttemptId, " + appAttemptId + ", is empty or null"); } - ApplicationAttemptId aaid = - ConverterUtils.toApplicationAttemptId(appAttemptId); + ApplicationAttemptId aaid = null; + try { + aaid = ConverterUtils.toApplicationAttemptId(appAttemptId); + } catch (Exception e) { + throw new BadRequestException(e); + } if (aaid == null) { throw new NotFoundException("appAttemptId is null"); } @@ -455,7 +464,12 @@ public class WebServices { throw new NotFoundException("containerId, " + containerId + ", is empty or null"); } - ContainerId cid = ConverterUtils.toContainerId(containerId); + ContainerId cid = null; + try { + cid = ConverterUtils.toContainerId(containerId); + } catch (Exception e) { + throw new BadRequestException(e); + } if (cid == null) { throw new NotFoundException("containerId is null"); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5092c941/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index de6d219..fddeb04 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.NotFoundException; import org.apache.hadoop.yarn.webapp.WebApp; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -143,10 +144,7 @@ public class NMWebServices { @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public AppInfo getNodeApp(@PathParam("appid") String appId) { init(); - ApplicationId id = ConverterUtils.toApplicationId(recordFactory, appId); - if (id == null) { - throw new NotFoundException("app with id " + appId + " not found"); - } + ApplicationId id = WebAppUtils.parseApplicationId(recordFactory, appId); Application app = this.nmContext.getApplications().get(id); if (app == null) { throw new NotFoundException("app with id " + appId + " not found"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5092c941/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java index e274abb..3cd8faf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java @@ -565,11 +565,14 @@ public class TestNMWebServicesApps extends JerseyTestBase { String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); WebServicesTestUtils.checkStringMatch("exception message", - "For input string: \"foo\"", message); + "java.lang.IllegalArgumentException: Invalid ApplicationId prefix: " + + "app_foo_0000. The valid ApplicationId should start with prefix" + + " application", + message); WebServicesTestUtils.checkStringMatch("exception type", - "NumberFormatException", type); + "BadRequestException", type); WebServicesTestUtils.checkStringMatch("exception classname", - "java.lang.NumberFormatException", classname); + "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5092c941/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 0ed8a4e..8036af4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -662,14 +662,7 @@ public class RMWebServices extends WebServices { public AppInfo getApp(@Context HttpServletRequest hsr, @PathParam("appid") String appId) { init(); - if (appId == null || appId.isEmpty()) { - throw new NotFoundException("appId, " + appId + ", is empty or null"); - } - ApplicationId id; - id = ConverterUtils.toApplicationId(recordFactory, appId); - if (id == null) { - throw new NotFoundException("appId is null"); - } + ApplicationId id = WebAppUtils.parseApplicationId(recordFactory, appId); RMApp app = rm.getRMContext().getRMApps().get(id); if (app == null) { throw new NotFoundException("app with id: " + appId + " not found"); @@ -684,14 +677,7 @@ public class RMWebServices extends WebServices { @PathParam("appid") String appId) { init(); - if (appId == null || appId.isEmpty()) { - throw new NotFoundException("appId, " + appId + ", is empty or null"); - } - ApplicationId id; - id = ConverterUtils.toApplicationId(recordFactory, appId); - if (id == null) { - throw new NotFoundException("appId is null"); - } + ApplicationId id = WebAppUtils.parseApplicationId(recordFactory, appId); RMApp app = rm.getRMContext().getRMApps().get(id); if (app == null) { throw new NotFoundException("app with id: " + appId + " not found"); @@ -1303,19 +1289,7 @@ public class RMWebServices extends WebServices { } private RMApp getRMAppForAppId(String appId) { - - if (appId == null || appId.isEmpty()) { - throw new NotFoundException("appId, " + appId + ", is empty or null"); - } - ApplicationId id; - try { - id = ConverterUtils.toApplicationId(recordFactory, appId); - } catch (NumberFormatException e) { - throw new NotFoundException("appId is invalid"); - } - if (id == null) { - throw new NotFoundException("appId is invalid"); - } + ApplicationId id = WebAppUtils.parseApplicationId(recordFactory, appId); RMApp app = rm.getRMContext().getRMApps().get(id); if (app == null) { throw new NotFoundException("app with id: " + appId + " not found"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5092c941/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 0328a7a..72cccf6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1194,11 +1194,13 @@ public class TestRMWebServicesApps extends JerseyTestBase { String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); WebServicesTestUtils.checkStringMatch("exception message", - "For input string: \"invalid\"", message); + "java.lang.IllegalArgumentException: Invalid ApplicationId:" + + " application_invalid_12", + message); WebServicesTestUtils.checkStringMatch("exception type", - "NumberFormatException", type); + "BadRequestException", type); WebServicesTestUtils.checkStringMatch("exception classname", - "java.lang.NumberFormatException", classname); + "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } finally { rm.stop(); @@ -1500,15 +1502,17 @@ public class TestRMWebServicesApps extends JerseyTestBase { public void testInvalidAppAttempts() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); + RMApp app = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); try { r.path("ws").path("v1").path("cluster").path("apps") - .path("application_invalid_12").accept(MediaType.APPLICATION_JSON) + .path(app.getApplicationId().toString()).path("appattempts") + .path("appattempt_invalid_12_000001") + .accept(MediaType.APPLICATION_JSON) .get(JSONObject.class); - fail("should have thrown exception on invalid appid"); + fail("should have thrown exception on invalid appAttempt"); } catch (UniformInterfaceException ue) { ClientResponse response = ue.getResponse(); @@ -1521,11 +1525,13 @@ public class TestRMWebServicesApps extends JerseyTestBase { String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); WebServicesTestUtils.checkStringMatch("exception message", - "For input string: \"invalid\"", message); + "java.lang.IllegalArgumentException: Invalid AppAttemptId:" + + " appattempt_invalid_12_000001", + message); WebServicesTestUtils.checkStringMatch("exception type", - "NumberFormatException", type); + "BadRequestException", type); WebServicesTestUtils.checkStringMatch("exception classname", - "java.lang.NumberFormatException", classname); + "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } finally { rm.stop(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5092c941/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java index 9abcf9c..38b32e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java @@ -568,17 +568,21 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); amNodeManager.nodeHeartbeat(true); String[] testAppIds = { "application_1391705042196_0001", "random_string" }; - for (String testAppId : testAppIds) { + for (int i = 0; i < testAppIds.length; i++) { AppState info = new AppState("KILLED"); ClientResponse response = - this.constructWebResource("apps", testAppId, "state") + this.constructWebResource("apps", testAppIds[i], "state") .accept(MediaType.APPLICATION_XML) .entity(info, MediaType.APPLICATION_XML).put(ClientResponse.class); if (!isAuthenticationEnabled()) { assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus()); continue; } - assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); + if (i == 0) { + assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); + } else { + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + } } rm.stop(); }
