YARN-6370. Properly handle rack requests for non-active subclusters in LocalityMulticastAMRMProxyPolicy. (Contributed by Botong Huang via curino).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/765f7a68 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/765f7a68 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/765f7a68 Branch: refs/heads/YARN-2915 Commit: 765f7a68fb5f08de3b5a81849aaa74c8e6b07cf7 Parents: d38a4de Author: Carlo Curino <cur...@apache.org> Authored: Wed Mar 22 13:53:47 2017 -0700 Committer: Subru Krishnan <su...@apache.org> Committed: Tue Jul 25 16:56:32 2017 -0700 ---------------------------------------------------------------------- .../LocalityMulticastAMRMProxyPolicy.java | 6 ++- .../TestLocalityMulticastAMRMProxyPolicy.java | 53 +++++++++++++------- 2 files changed, 41 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/765f7a68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java index 6f97a51..454962f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java @@ -261,7 +261,11 @@ public class LocalityMulticastAMRMProxyPolicy extends AbstractAMRMProxyPolicy { // If home-subcluster is not active, ignore node/rack request if (bookkeeper.isActiveAndEnabled(homeSubcluster)) { - bookkeeper.addLocalizedNodeRR(homeSubcluster, rr); + if (targetIds != null && targetIds.size() > 0) { + bookkeeper.addRackRR(homeSubcluster, rr); + } else { + bookkeeper.addLocalizedNodeRR(homeSubcluster, rr); + } } else { if (LOG.isDebugEnabled()) { LOG.debug("The homeSubCluster (" + homeSubcluster + ") we are " http://git-wip-us.apache.org/repos/asf/hadoop/blob/765f7a68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java index 5b3cf74..6e3a2f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java @@ -339,19 +339,20 @@ public class TestLocalityMulticastAMRMProxyPolicy validateSplit(response, resourceRequests); prettyPrintRequests(response); - // we expect 4 entry for home subcluster (3 for request-id 4, and a part - // of the broadcast of request-id 2 - checkExpectedAllocation(response, getHomeSubCluster().getId(), 4, 23); + // we expect 7 entries for home subcluster (2 for request-id 4, 3 for + // request-id 5, and a part of the broadcast of request-id 2 + checkExpectedAllocation(response, getHomeSubCluster().getId(), 7, 29); - // for subcluster0 we expect 3 entry from request-id 0, and 3 from - // request-id 3, as well as part of the request-id 2 broadast - checkExpectedAllocation(response, "subcluster0", 7, 26); + // for subcluster0 we expect 10 entries, 3 from request-id 0, and 3 from + // request-id 3, 3 entries from request-id 5, as well as part of the + // request-id 2 broadast + checkExpectedAllocation(response, "subcluster0", 10, 32); - // we expect 5 entry for subcluster1 (4 from request-id 1, and part + // we expect 5 entries for subcluster1 (4 from request-id 1, and part // of the broadcast of request-id 2 checkExpectedAllocation(response, "subcluster1", 5, 26); - // sub-cluster 2 should contain 3 entry from request-id 1 and 1 from the + // sub-cluster 2 should contain 3 entries from request-id 1 and 1 from the // broadcast of request-id 2, and no request-id 0 checkExpectedAllocation(response, "subcluster2", 4, 23); @@ -364,28 +365,33 @@ public class TestLocalityMulticastAMRMProxyPolicy // check that the allocations that show up are what expected for (ResourceRequest rr : response.get(getHomeSubCluster())) { - Assert.assertTrue(rr.getAllocationRequestId() == 4L - || rr.getAllocationRequestId() == 2L); - } - - for (ResourceRequest rr : response.get(getHomeSubCluster())) { - Assert.assertTrue(rr.getAllocationRequestId() != 1L); + Assert.assertTrue( + rr.getAllocationRequestId() == 2L || rr.getAllocationRequestId() == 4L + || rr.getAllocationRequestId() == 5L); } List<ResourceRequest> rrs = response.get(SubClusterId.newInstance("subcluster0")); for (ResourceRequest rr : rrs) { Assert.assertTrue(rr.getAllocationRequestId() != 1L); + Assert.assertTrue(rr.getAllocationRequestId() != 4L); + } + + for (ResourceRequest rr : response + .get(SubClusterId.newInstance("subcluster1"))) { + Assert.assertTrue(rr.getAllocationRequestId() == 1L + || rr.getAllocationRequestId() == 2L); } for (ResourceRequest rr : response .get(SubClusterId.newInstance("subcluster2"))) { - Assert.assertTrue(rr.getAllocationRequestId() != 0L); + Assert.assertTrue(rr.getAllocationRequestId() == 1L + || rr.getAllocationRequestId() == 2L); } for (ResourceRequest rr : response .get(SubClusterId.newInstance("subcluster5"))) { - Assert.assertTrue(rr.getAllocationRequestId() >= 2); + Assert.assertTrue(rr.getAllocationRequestId() == 2); Assert.assertTrue(rr.getRelaxLocality()); } } @@ -555,7 +561,7 @@ public class TestLocalityMulticastAMRMProxyPolicy out.add(FederationPoliciesTestUtil.createResourceRequest(1L, "subcluster2-rack3", 1024, 1, 1, 1, null, false)); out.add(FederationPoliciesTestUtil.createResourceRequest(1L, - ResourceRequest.ANY, 1024, 1, 1, 2, null, false)); + ResourceRequest.ANY, 1024, 1, 1, 3, null, false)); // create a non-local ANY request that can span anything out.add(FederationPoliciesTestUtil.createResourceRequest(2L, @@ -578,6 +584,19 @@ public class TestLocalityMulticastAMRMProxyPolicy out.add(FederationPoliciesTestUtil.createResourceRequest(4L, ResourceRequest.ANY, 1024, 1, 1, 1, null, false)); + // create a request of two hosts, an unknown node and a known node, both in + // a known rack, and expect the unknown node to show up in homesubcluster + out.add(FederationPoliciesTestUtil.createResourceRequest(5L, + "subcluster0-rack0-host0", 1024, 1, 1, 2, null, false)); + out.add(FederationPoliciesTestUtil.createResourceRequest(5L, + "subcluster0-rack0", 1024, 1, 1, 2, null, false)); + out.add(FederationPoliciesTestUtil.createResourceRequest(5L, "node4", 1024, + 1, 1, 2, null, false)); + out.add(FederationPoliciesTestUtil.createResourceRequest(5L, "rack2", 1024, + 1, 1, 2, null, false)); + out.add(FederationPoliciesTestUtil.createResourceRequest(5L, + ResourceRequest.ANY, 1024, 1, 1, 4, null, false)); + return out; } } \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
