YARN-5408. Compose Federation membership/application/policy APIs into an uber FederationStateStore API. (Ellen Hui via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e6bf759 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e6bf759 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e6bf759 Branch: refs/heads/YARN-2915 Commit: 3e6bf7596c88ce6c54fe35d1d9869e7ec48e8f92 Parents: 04ff565 Author: Subru Krishnan <su...@apache.org> Authored: Mon Aug 8 14:53:38 2016 -0700 Committer: Subru Krishnan <su...@apache.org> Committed: Tue Nov 22 18:37:54 2016 -0800 ---------------------------------------------------------------------- ...ederationApplicationHomeSubClusterStore.java | 18 ++---- .../store/FederationMembershipStateStore.java | 14 +---- .../federation/store/FederationStateStore.java | 64 ++++++++++++++++++++ .../store/impl/MemoryFederationStateStore.java | 19 ------ .../impl/FederationStateStoreBaseTest.java | 57 +++++++++-------- .../impl/TestMemoryFederationStateStore.java | 21 +------ 6 files changed, 99 insertions(+), 94 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e6bf759/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java index 217ee2e..22bb88a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java @@ -30,7 +30,6 @@ import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHom import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterResponse; import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse; -import org.apache.hadoop.yarn.server.records.Version; /** * FederationApplicationHomeSubClusterStore maintains the state of all @@ -50,15 +49,6 @@ import org.apache.hadoop.yarn.server.records.Version; public interface FederationApplicationHomeSubClusterStore { /** - * Get the {@link Version} of the underlying federation application state - * store. - * - * @return the {@link Version} of the underlying federation application state - * store - */ - Version getApplicationStateStoreVersion(); - - /** * Register the home {@code SubClusterId} of the newly submitted * {@code ApplicationId}. Currently response is empty if the operation was * successful, if not an exception reporting reason for a failure. @@ -91,16 +81,16 @@ public interface FederationApplicationHomeSubClusterStore { * {@code ApplicationId}. * * @param request contains the application queried - * @return {@code ApplicationHomeSubCluster} containing the application's - * home subcluster + * @return {@code ApplicationHomeSubCluster} containing the application's home + * subcluster * @throws YarnException if the request is invalid/fails */ GetApplicationHomeSubClusterResponse getApplicationHomeSubClusterMap( GetApplicationHomeSubClusterRequest request) throws YarnException; /** - * Get the {@code ApplicationHomeSubCluster} list representing the mapping - * of all submitted applications to it's home sub-cluster. + * Get the {@code ApplicationHomeSubCluster} list representing the mapping of + * all submitted applications to it's home sub-cluster. * * @param request empty representing all applications * @return the mapping of all submitted application to it's home sub-cluster http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e6bf759/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java index 378eadc..7778d5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java @@ -30,7 +30,6 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbea import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatResponse; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse; -import org.apache.hadoop.yarn.server.records.Version; /** * FederationMembershipStateStore maintains the state of all @@ -42,15 +41,6 @@ import org.apache.hadoop.yarn.server.records.Version; public interface FederationMembershipStateStore { /** - * Get the {@link Version} of the underlying federation membership state - * store. - * - * @return the {@link Version} of the underlying federation membership state - * store - */ - Version getMembershipStateStoreVersion(); - - /** * Register a <em>subcluster</em> by publishing capabilities as represented by * {@code SubClusterInfo} to indicate participation in federation. This is * typically done during initialization or restart/failover of the @@ -80,7 +70,7 @@ public interface FederationMembershipStateStore { */ SubClusterDeregisterResponse deregisterSubCluster( SubClusterDeregisterRequest subClusterDeregisterRequest) - throws YarnException; + throws YarnException; /** * Periodic heartbeat from a <code>ResourceManager</code> participating in @@ -96,7 +86,7 @@ public interface FederationMembershipStateStore { */ SubClusterHeartbeatResponse subClusterHeartbeat( SubClusterHeartbeatRequest subClusterHeartbeatRequest) - throws YarnException; + throws YarnException; /** * Get the membership information of <em>subcluster</em> as identified by http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e6bf759/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java new file mode 100644 index 0000000..9397e9c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.federation.store; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.records.Version; + +/** + * FederationStore extends the three interfaces used to coordinate the state of + * a federated cluster: {@link FederationApplicationHomeSubClusterStore}, + * {@link FederationMembershipStateStore}, and {@link FederationPolicyStore}. + * + */ +public interface FederationStateStore + extends FederationApplicationHomeSubClusterStore, + FederationMembershipStateStore, FederationPolicyStore { + + /** + * Initialize the FederationStore. + * + * @param conf the cluster configuration + * @throws YarnException if initialization fails + */ + void init(Configuration conf) throws YarnException; + + /** + * Perform any cleanup operations of the StateStore. + * + * @throws Exception if cleanup fails + */ + void close() throws Exception; + + /** + * Get the {@link Version} of the underlying federation state store client. + * + * @return the {@link Version} of the underlying federation store client + */ + Version getCurrentVersion(); + + /** + * Load the version information from the federation state store. + * + * @return the {@link Version} of the federation state store + */ + Version loadVersion(); + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e6bf759/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java index 7fdc4a9..cea4ac2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java @@ -36,11 +36,8 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbea import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatResponse; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse; -import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.util.MonotonicClock; -import com.google.common.annotations.VisibleForTesting; - /** * In-memory implementation of FederationMembershipStateStore. */ @@ -52,11 +49,6 @@ public class MemoryFederationStateStore private final MonotonicClock clock = new MonotonicClock(); @Override - public Version getMembershipStateStoreVersion() { - return null; - } - - @Override public SubClusterRegisterResponse registerSubCluster( SubClusterRegisterRequest request) throws YarnException { SubClusterInfo subClusterInfo = request.getSubClusterInfo(); @@ -122,17 +114,6 @@ public class MemoryFederationStateStore } } return GetSubClustersInfoResponse.newInstance(result); - - } - - @VisibleForTesting - public Map<SubClusterId, SubClusterInfo> getMembershipTable() { - return membership; - } - - @VisibleForTesting - public void clearMembershipTable() { - membership.clear(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e6bf759/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java index 7eb1c86..c76a485 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java @@ -19,25 +19,21 @@ package org.apache.hadoop.yarn.server.federation.store.impl; import java.io.IOException; -import org.junit.Before; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Map; - -import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; -import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; -import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState; -import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.store.FederationMembershipStateStore; import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest; import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState; import org.apache.hadoop.yarn.util.MonotonicClock; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; /** @@ -45,17 +41,22 @@ import org.junit.Test; */ public abstract class FederationStateStoreBaseTest { - static final Logger LOG = - LoggerFactory.getLogger(FederationStateStoreBaseTest.class); private static final MonotonicClock CLOCK = new MonotonicClock(); - private FederationMembershipStateStore stateStore = getStateStore(); + private FederationMembershipStateStore stateStore; @Before public void before() throws IOException { - clearMembership(); + stateStore = getCleanStateStore(); } + @After + public void after() { + stateStore = null; + } + + protected abstract FederationMembershipStateStore getCleanStateStore(); + @Test public void testRegisterSubCluster() throws Exception { SubClusterId subClusterId = SubClusterId.newInstance("SC"); @@ -63,11 +64,9 @@ public abstract class FederationStateStoreBaseTest { SubClusterRegisterResponse result = stateStore.registerSubCluster( SubClusterRegisterRequest.newInstance(subClusterInfo)); - Map<SubClusterId, SubClusterInfo> membership = getMembership(); - Assert.assertNotNull(membership.get(subClusterId)); Assert.assertNotNull(result); - Assert.assertEquals(subClusterInfo, membership.get(subClusterId)); + Assert.assertEquals(subClusterInfo, querySubClusterInfo(subClusterId)); } @Test @@ -83,10 +82,8 @@ public abstract class FederationStateStoreBaseTest { stateStore.deregisterSubCluster(deregisterRequest); - Map<SubClusterId, SubClusterInfo> membership = getMembership(); - Assert.assertNotNull(membership.get(subClusterId)); - Assert.assertEquals(membership.get(subClusterId).getState(), - SubClusterState.SC_UNREGISTERED); + Assert.assertEquals(SubClusterState.SC_UNREGISTERED, + querySubClusterInfo(subClusterId).getState()); } @Test @@ -179,10 +176,9 @@ public abstract class FederationStateStoreBaseTest { .newInstance(subClusterId, SubClusterState.SC_RUNNING, "cabability"); stateStore.subClusterHeartbeat(heartbeatRequest); - Map<SubClusterId, SubClusterInfo> membership = getMembership(); - Assert.assertEquals(membership.get(subClusterId).getState(), - SubClusterState.SC_RUNNING); - Assert.assertNotNull(membership.get(subClusterId).getLastHeartBeat()); + Assert.assertEquals(SubClusterState.SC_RUNNING, + querySubClusterInfo(subClusterId).getState()); + Assert.assertNotNull(querySubClusterInfo(subClusterId).getLastHeartBeat()); } @Test @@ -212,10 +208,11 @@ public abstract class FederationStateStoreBaseTest { CLOCK.getTime(), "cabability"); } - protected abstract Map<SubClusterId, SubClusterInfo> getMembership(); - - protected abstract void clearMembership(); - - protected abstract FederationMembershipStateStore getStateStore(); + private SubClusterInfo querySubClusterInfo(SubClusterId subClusterId) + throws YarnException { + GetSubClusterInfoRequest request = + GetSubClusterInfoRequest.newInstance(subClusterId); + return stateStore.getSubCluster(request).getSubClusterInfo(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e6bf759/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java index b74ffbd..9396eda 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java @@ -17,11 +17,7 @@ package org.apache.hadoop.yarn.server.federation.store.impl; -import java.util.Map; - import org.apache.hadoop.yarn.server.federation.store.FederationMembershipStateStore; -import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; -import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; /** * Unit tests for MemoryFederationStateStore. @@ -29,21 +25,8 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; public class TestMemoryFederationStateStore extends FederationStateStoreBaseTest { - private static final MemoryFederationStateStore STATESTORE = - new MemoryFederationStateStore(); - - @Override - protected Map<SubClusterId, SubClusterInfo> getMembership() { - return STATESTORE.getMembershipTable(); - } - - @Override - protected void clearMembership() { - STATESTORE.clearMembershipTable(); - } - @Override - protected FederationMembershipStateStore getStateStore() { - return STATESTORE; + protected FederationMembershipStateStore getCleanStateStore() { + return new MemoryFederationStateStore(); } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
