http://git-wip-us.apache.org/repos/asf/hadoop/blob/16fa876c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java index 4975a9f..5fa02d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java @@ -29,6 +29,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo; import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException; +import org.apache.hadoop.yarn.server.federation.policies.manager.PriorityBroadcastPolicyManager; +import org.apache.hadoop.yarn.server.federation.policies.manager.UniformBroadcastPolicyManager; import org.apache.hadoop.yarn.server.federation.policies.router.PriorityRouterPolicy; import org.apache.hadoop.yarn.server.federation.policies.router.UniformRandomRouterPolicy; import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16fa876c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java deleted file mode 100644 index 542a5ae..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.yarn.server.federation.policies; - -import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy; -import org.apache.hadoop.yarn.server.federation.policies.router.UniformRandomRouterPolicy; -import org.junit.Before; - -/** - * Simple test of {@link UniformBroadcastPolicyManager}. - */ -public class TestUniformBroadcastPolicyManager extends BasePolicyManagerTest { - - @Before - public void setup() { - //config policy - wfp = new UniformBroadcastPolicyManager(); - wfp.setQueue("queue1"); - - //set expected params that the base test class will use for tests - expectedPolicyManager = UniformBroadcastPolicyManager.class; - expectedAMRMProxyPolicy = BroadcastAMRMProxyPolicy.class; - expectedRouterPolicy = UniformRandomRouterPolicy.class; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/16fa876c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestWeightedLocalityPolicyManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestWeightedLocalityPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestWeightedLocalityPolicyManager.java deleted file mode 100644 index ab9cec4..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestWeightedLocalityPolicyManager.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.yarn.server.federation.policies; - -import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.LocalityMulticastAMRMProxyPolicy; -import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo; -import org.apache.hadoop.yarn.server.federation.policies.router.WeightedRandomRouterPolicy; -import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; -import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.util.HashMap; -import java.util.Map; - -/** - * Simple test of {@link WeightedLocalityPolicyManager}. - */ -public class TestWeightedLocalityPolicyManager extends - BasePolicyManagerTest { - - private WeightedPolicyInfo policyInfo; - - @Before - public void setup() { - // configure a policy - - wfp = new WeightedLocalityPolicyManager(); - wfp.setQueue("queue1"); - SubClusterId sc1 = SubClusterId.newInstance("sc1"); - SubClusterId sc2 = SubClusterId.newInstance("sc2"); - policyInfo = new WeightedPolicyInfo(); - - Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>(); - routerWeights.put(new SubClusterIdInfo(sc1), 0.2f); - routerWeights.put(new SubClusterIdInfo(sc2), 0.8f); - policyInfo.setRouterPolicyWeights(routerWeights); - - Map<SubClusterIdInfo, Float> amrmWeights = new HashMap<>(); - amrmWeights.put(new SubClusterIdInfo(sc1), 0.2f); - amrmWeights.put(new SubClusterIdInfo(sc2), 0.8f); - policyInfo.setAMRMPolicyWeights(amrmWeights); - - ((WeightedLocalityPolicyManager) wfp).setWeightedPolicyInfo( - policyInfo); - - //set expected params that the base test class will use for tests - expectedPolicyManager = WeightedLocalityPolicyManager.class; - expectedAMRMProxyPolicy = LocalityMulticastAMRMProxyPolicy.class; - expectedRouterPolicy = WeightedRandomRouterPolicy.class; - } - - @Test - public void testPolicyInfoSetCorrectly() throws Exception { - serializeAndDeserializePolicyManager(wfp, expectedPolicyManager, - expectedAMRMProxyPolicy, - expectedRouterPolicy); - - //check the policyInfo propagates through ser/der correctly - Assert.assertEquals(((WeightedLocalityPolicyManager) wfp) - .getWeightedPolicyInfo(), policyInfo); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/16fa876c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/BasePolicyManagerTest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/BasePolicyManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/BasePolicyManagerTest.java new file mode 100644 index 0000000..3cf73b6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/BasePolicyManagerTest.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.policies.manager; + +import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext; +import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy; +import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException; +import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration; +import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil; +import org.junit.Assert; +import org.junit.Test; + +/** + * This class provides common test methods for testing {@code + * FederationPolicyManager}s. + */ +public abstract class BasePolicyManagerTest { + + @SuppressWarnings("checkstyle:visibilitymodifier") + protected FederationPolicyManager wfp = null; + @SuppressWarnings("checkstyle:visibilitymodifier") + protected Class expectedPolicyManager; + @SuppressWarnings("checkstyle:visibilitymodifier") + protected Class expectedAMRMProxyPolicy; + @SuppressWarnings("checkstyle:visibilitymodifier") + protected Class expectedRouterPolicy; + + @Test + public void testSerializeAndInstantiate() throws Exception { + serializeAndDeserializePolicyManager(wfp, expectedPolicyManager, + expectedAMRMProxyPolicy, expectedRouterPolicy); + } + + @Test(expected = FederationPolicyInitializationException.class) + public void testSerializeAndInstantiateBad1() throws Exception { + serializeAndDeserializePolicyManager(wfp, String.class, + expectedAMRMProxyPolicy, expectedRouterPolicy); + } + + @Test(expected = AssertionError.class) + public void testSerializeAndInstantiateBad2() throws Exception { + serializeAndDeserializePolicyManager(wfp, expectedPolicyManager, + String.class, expectedRouterPolicy); + } + + @Test(expected = AssertionError.class) + public void testSerializeAndInstantiateBad3() throws Exception { + serializeAndDeserializePolicyManager(wfp, expectedPolicyManager, + expectedAMRMProxyPolicy, String.class); + } + + protected static void serializeAndDeserializePolicyManager( + FederationPolicyManager wfp, Class policyManagerType, + Class expAMRMProxyPolicy, Class expRouterPolicy) throws Exception { + + // serializeConf it in a context + SubClusterPolicyConfiguration fpc = wfp.serializeConf(); + fpc.setType(policyManagerType.getCanonicalName()); + FederationPolicyInitializationContext context = + new FederationPolicyInitializationContext(); + context.setSubClusterPolicyConfiguration(fpc); + context + .setFederationStateStoreFacade(FederationPoliciesTestUtil.initFacade()); + context.setFederationSubclusterResolver( + FederationPoliciesTestUtil.initResolver()); + context.setHomeSubcluster(SubClusterId.newInstance("homesubcluster")); + + // based on the "context" created instantiate new class and use it + Class c = Class.forName(wfp.getClass().getCanonicalName()); + FederationPolicyManager wfp2 = (FederationPolicyManager) c.newInstance(); + + FederationAMRMProxyPolicy federationAMRMProxyPolicy = + wfp2.getAMRMPolicy(context, null); + + // needed only for tests (getARMRMPolicy change the "type" in conf) + fpc.setType(wfp.getClass().getCanonicalName()); + + FederationRouterPolicy federationRouterPolicy = + wfp2.getRouterPolicy(context, null); + + Assert.assertEquals(federationAMRMProxyPolicy.getClass(), + expAMRMProxyPolicy); + + Assert.assertEquals(federationRouterPolicy.getClass(), expRouterPolicy); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/16fa876c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestHashBasedBroadcastPolicyManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestHashBasedBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestHashBasedBroadcastPolicyManager.java new file mode 100644 index 0000000..5fc4a56 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestHashBasedBroadcastPolicyManager.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.policies.manager; + +import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy; +import org.apache.hadoop.yarn.server.federation.policies.router.HashBasedRouterPolicy; +import org.junit.Before; + +/** + * Simple test of {@link HashBroadcastPolicyManager}. + */ +public class TestHashBasedBroadcastPolicyManager extends BasePolicyManagerTest { + + @Before + public void setup() { + // config policy + wfp = new HashBroadcastPolicyManager(); + wfp.setQueue("queue1"); + + // set expected params that the base test class will use for tests + expectedPolicyManager = HashBroadcastPolicyManager.class; + expectedAMRMProxyPolicy = BroadcastAMRMProxyPolicy.class; + expectedRouterPolicy = HashBasedRouterPolicy.class; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/16fa876c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestPriorityBroadcastPolicyManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestPriorityBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestPriorityBroadcastPolicyManager.java new file mode 100644 index 0000000..21b39e9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestPriorityBroadcastPolicyManager.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.policies.manager; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy; +import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo; +import org.apache.hadoop.yarn.server.federation.policies.router.PriorityRouterPolicy; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +/** + * Simple test of {@link PriorityBroadcastPolicyManager}. + */ +public class TestPriorityBroadcastPolicyManager extends BasePolicyManagerTest { + + private WeightedPolicyInfo policyInfo; + + @Before + public void setup() { + // configure a policy + + wfp = new PriorityBroadcastPolicyManager(); + wfp.setQueue("queue1"); + SubClusterId sc1 = SubClusterId.newInstance("sc1"); + SubClusterId sc2 = SubClusterId.newInstance("sc2"); + policyInfo = new WeightedPolicyInfo(); + + Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>(); + routerWeights.put(new SubClusterIdInfo(sc1), 0.2f); + routerWeights.put(new SubClusterIdInfo(sc2), 0.8f); + policyInfo.setRouterPolicyWeights(routerWeights); + + ((PriorityBroadcastPolicyManager) wfp).setWeightedPolicyInfo(policyInfo); + + // set expected params that the base test class will use for tests + expectedPolicyManager = PriorityBroadcastPolicyManager.class; + expectedAMRMProxyPolicy = BroadcastAMRMProxyPolicy.class; + expectedRouterPolicy = PriorityRouterPolicy.class; + } + + @Test + public void testPolicyInfoSetCorrectly() throws Exception { + serializeAndDeserializePolicyManager(wfp, expectedPolicyManager, + expectedAMRMProxyPolicy, expectedRouterPolicy); + + // check the policyInfo propagates through ser/der correctly + Assert.assertEquals( + ((PriorityBroadcastPolicyManager) wfp).getWeightedPolicyInfo(), + policyInfo); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/16fa876c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestUniformBroadcastPolicyManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestUniformBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestUniformBroadcastPolicyManager.java new file mode 100644 index 0000000..57fafdc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestUniformBroadcastPolicyManager.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.policies.manager; + +import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy; +import org.apache.hadoop.yarn.server.federation.policies.router.UniformRandomRouterPolicy; +import org.junit.Before; + +/** + * Simple test of {@link UniformBroadcastPolicyManager}. + */ +public class TestUniformBroadcastPolicyManager extends BasePolicyManagerTest { + + @Before + public void setup() { + //config policy + wfp = new UniformBroadcastPolicyManager(); + wfp.setQueue("queue1"); + + //set expected params that the base test class will use for tests + expectedPolicyManager = UniformBroadcastPolicyManager.class; + expectedAMRMProxyPolicy = BroadcastAMRMProxyPolicy.class; + expectedRouterPolicy = UniformRandomRouterPolicy.class; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/16fa876c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestWeightedLocalityPolicyManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestWeightedLocalityPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestWeightedLocalityPolicyManager.java new file mode 100644 index 0000000..51661473 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestWeightedLocalityPolicyManager.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.policies.manager; + +import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.LocalityMulticastAMRMProxyPolicy; +import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo; +import org.apache.hadoop.yarn.server.federation.policies.router.WeightedRandomRouterPolicy; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +/** + * Simple test of {@link WeightedLocalityPolicyManager}. + */ +public class TestWeightedLocalityPolicyManager extends + BasePolicyManagerTest { + + private WeightedPolicyInfo policyInfo; + + @Before + public void setup() { + // configure a policy + + wfp = new WeightedLocalityPolicyManager(); + wfp.setQueue("queue1"); + SubClusterId sc1 = SubClusterId.newInstance("sc1"); + SubClusterId sc2 = SubClusterId.newInstance("sc2"); + policyInfo = new WeightedPolicyInfo(); + + Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>(); + routerWeights.put(new SubClusterIdInfo(sc1), 0.2f); + routerWeights.put(new SubClusterIdInfo(sc2), 0.8f); + policyInfo.setRouterPolicyWeights(routerWeights); + + Map<SubClusterIdInfo, Float> amrmWeights = new HashMap<>(); + amrmWeights.put(new SubClusterIdInfo(sc1), 0.2f); + amrmWeights.put(new SubClusterIdInfo(sc2), 0.8f); + policyInfo.setAMRMPolicyWeights(amrmWeights); + + ((WeightedLocalityPolicyManager) wfp).setWeightedPolicyInfo( + policyInfo); + + //set expected params that the base test class will use for tests + expectedPolicyManager = WeightedLocalityPolicyManager.class; + expectedAMRMProxyPolicy = LocalityMulticastAMRMProxyPolicy.class; + expectedRouterPolicy = WeightedRandomRouterPolicy.class; + } + + @Test + public void testPolicyInfoSetCorrectly() throws Exception { + serializeAndDeserializePolicyManager(wfp, expectedPolicyManager, + expectedAMRMProxyPolicy, + expectedRouterPolicy); + + //check the policyInfo propagates through ser/der correctly + Assert.assertEquals(((WeightedLocalityPolicyManager) wfp) + .getWeightedPolicyInfo(), policyInfo); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/16fa876c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseRouterPoliciesTest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseRouterPoliciesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseRouterPoliciesTest.java new file mode 100644 index 0000000..2e7a0af --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseRouterPoliciesTest.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.federation.policies.router; + +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest; +import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.junit.Assert; +import org.junit.Test; + +/** + * Base class for router policies tests, tests for null input cases. + */ +public abstract class BaseRouterPoliciesTest + extends BaseFederationPoliciesTest { + + @Test + public void testNullQueueRouting() throws YarnException { + FederationRouterPolicy localPolicy = (FederationRouterPolicy) getPolicy(); + ApplicationSubmissionContext applicationSubmissionContext = + ApplicationSubmissionContext.newInstance(null, null, null, null, null, + false, false, 0, Resources.none(), null, false, null, null); + SubClusterId chosen = + localPolicy.getHomeSubcluster(applicationSubmissionContext); + Assert.assertNotNull(chosen); + } + + @Test(expected = FederationPolicyException.class) + public void testNullAppContext() throws YarnException { + ((FederationRouterPolicy) getPolicy()).getHomeSubcluster(null); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/16fa876c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestHashBasedRouterPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestHashBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestHashBasedRouterPolicy.java new file mode 100644 index 0000000..af7fe43 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestHashBasedRouterPolicy.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.policies.router; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +/** + * Simple test class for the {@link HashBasedRouterPolicy}. Tests that one of + * the active sub-cluster is chosen. + */ +public class TestHashBasedRouterPolicy extends BaseRouterPoliciesTest { + + private int numSubclusters = 10; + + @Before + public void setUp() throws Exception { + + // set policy in base class + setPolicy(new HashBasedRouterPolicy()); + + // setting up the active sub-clusters for this test + setMockActiveSubclusters(numSubclusters); + + // initialize policy with context + FederationPoliciesTestUtil.initializePolicyContext(getPolicy(), + getPolicyInfo(), getActiveSubclusters()); + } + + @Test + public void testHashSpreadUniformlyAmongSubclusters() throws YarnException { + SubClusterId chosen; + + Map<SubClusterId, AtomicLong> counter = new HashMap<>(); + for (SubClusterId id : getActiveSubclusters().keySet()) { + counter.put(id, new AtomicLong(0)); + } + + long jobPerSub = 100; + + ApplicationSubmissionContext applicationSubmissionContext = + mock(ApplicationSubmissionContext.class); + for (int i = 0; i < jobPerSub * numSubclusters; i++) { + when(applicationSubmissionContext.getQueue()).thenReturn("queue" + i); + chosen = ((FederationRouterPolicy) getPolicy()) + .getHomeSubcluster(applicationSubmissionContext); + counter.get(chosen).addAndGet(1); + } + + // hash spread the jobs equally among the subclusters + for (AtomicLong a : counter.values()) { + Assert.assertEquals(a.get(), jobPerSub); + } + + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/16fa876c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java index 906e35f..b70b4aa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java @@ -21,7 +21,6 @@ import java.util.HashMap; import java.util.Map; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest; import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; @@ -36,7 +35,7 @@ import org.junit.Test; * Simple test class for the {@link LoadBasedRouterPolicy}. Test that the load * is properly considered for allocation. */ -public class TestLoadBasedRouterPolicy extends BaseFederationPoliciesTest { +public class TestLoadBasedRouterPolicy extends BaseRouterPoliciesTest { @Before public void setUp() throws Exception { http://git-wip-us.apache.org/repos/asf/hadoop/blob/16fa876c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java index eefcfd9..42d919d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java @@ -23,7 +23,6 @@ import java.util.HashMap; import java.util.Map; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest; import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; @@ -38,7 +37,7 @@ import org.junit.Test; * Simple test class for the {@link PriorityRouterPolicy}. Tests that the * weights are correctly used for ordering the choice of sub-clusters. */ -public class TestPriorityRouterPolicy extends BaseFederationPoliciesTest { +public class TestPriorityRouterPolicy extends BaseRouterPoliciesTest { @Before public void setUp() throws Exception { http://git-wip-us.apache.org/repos/asf/hadoop/blob/16fa876c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java index ac41ab5..b45aa2a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java @@ -21,7 +21,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest; import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; @@ -36,7 +35,7 @@ import org.junit.Test; * Simple test class for the {@link UniformRandomRouterPolicy}. Tests that one * of the active subcluster is chosen. */ -public class TestUniformRandomRouterPolicy extends BaseFederationPoliciesTest { +public class TestUniformRandomRouterPolicy extends BaseRouterPoliciesTest { @Before public void setUp() throws Exception { http://git-wip-us.apache.org/repos/asf/hadoop/blob/16fa876c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java index 78967d0..34cc5f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java @@ -24,8 +24,8 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest; import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; @@ -41,7 +41,7 @@ import org.junit.Test; * number of randomized tests to check we are weighiting correctly even if * clusters go inactive. */ -public class TestWeightedRandomRouterPolicy extends BaseFederationPoliciesTest { +public class TestWeightedRandomRouterPolicy extends BaseRouterPoliciesTest { @Before public void setUp() throws Exception { @@ -78,13 +78,18 @@ public class TestWeightedRandomRouterPolicy extends BaseFederationPoliciesTest { @Test public void testClusterChosenWithRightProbability() throws YarnException { + ApplicationSubmissionContext context = + mock(ApplicationSubmissionContext.class); + when(context.getQueue()).thenReturn("queue1"); + setApplicationSubmissionContext(context); + Map<SubClusterId, AtomicLong> counter = new HashMap<>(); for (SubClusterIdInfo id : getPolicyInfo().getRouterPolicyWeights() .keySet()) { counter.put(id.toId(), new AtomicLong(0)); } - float numberOfDraws = 1000000; + float numberOfDraws = 100000; for (float i = 0; i < numberOfDraws; i++) { SubClusterId chosenId = ((FederationRouterPolicy) getPolicy()) @@ -113,8 +118,8 @@ public class TestWeightedRandomRouterPolicy extends BaseFederationPoliciesTest { Assert.assertTrue( "Id " + counterEntry.getKey() + " Actual weight: " + actualWeight + " expected weight: " + expectedWeight, - expectedWeight == 0 || (actualWeight / expectedWeight) < 1.1 - && (actualWeight / expectedWeight) > 0.9); + expectedWeight == 0 || (actualWeight / expectedWeight) < 1.2 + && (actualWeight / expectedWeight) > 0.8); } else { Assert .assertTrue( --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org