This is an automated email from the ASF dual-hosted git repository. nanda pushed a commit to branch HDDS-2823 in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit a70964ec84ac04ca3f0dbad9dd7b43f0f0807707 Author: Glen Geng <gleng...@tencent.com> AuthorDate: Sat Oct 24 21:04:32 2020 +0530 HDDS-4130. remove the 1st edition of RatisServer of SCM HA which is copied from OM HA. --- .../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 14 -- .../hdds/scm/server/StorageContainerManager.java | 78 +---------- .../hdds/scm/server/ratis/SCMStateMachine.java | 6 +- .../hdds/scm/server/ratis/TestSCMRatisServer.java | 147 --------------------- .../hdds/scm/server/ratis/TestSCMStateMachine.java | 120 ----------------- 5 files changed, 5 insertions(+), 360 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java index 0f71744..48946b4 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java @@ -19,13 +19,9 @@ package org.apache.hadoop.hdds.scm.ha; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.scm.ScmConfigKeys; -import org.apache.hadoop.hdds.scm.ScmUtils; -import org.apache.hadoop.hdds.scm.server.ratis.SCMRatisServer; -import java.io.File; import java.util.Collection; /** @@ -42,16 +38,6 @@ public final class SCMHAUtils { ScmConfigKeys.OZONE_SCM_HA_ENABLE_DEFAULT); } - public static File createSCMRatisDir(ConfigurationSource conf) - throws IllegalArgumentException { - String scmRatisDir = SCMRatisServer.getSCMRatisDirectory(conf); - if (scmRatisDir == null || scmRatisDir.isEmpty()) { - throw new IllegalArgumentException(HddsConfigKeys.OZONE_METADATA_DIRS + - " must be defined."); - } - return ScmUtils.createSCMDir(scmRatisDir); - } - /** * Get a collection of all scmNodeIds for the given scmServiceId. */ diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index cbd713c..b17729b 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -31,13 +31,11 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.RemovalListener; import com.google.protobuf.BlockingService; -import java.io.File; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.util.Collection; import java.util.HashMap; import java.util.Map; -import java.util.Collections; import java.util.Objects; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; @@ -54,10 +52,6 @@ import org.apache.hadoop.hdds.scm.PipelineChoosePolicy; import org.apache.hadoop.hdds.scm.PlacementPolicy; import org.apache.hadoop.hdds.scm.ha.SCMHAManager; import org.apache.hadoop.hdds.scm.ha.SCMHAManagerImpl; -import org.apache.hadoop.hdds.scm.ha.SCMHAUtils; -import org.apache.hadoop.hdds.scm.ha.SCMNodeDetails; -import org.apache.hadoop.hdds.scm.server.ratis.SCMRatisServer; -import org.apache.hadoop.hdds.scm.server.ratis.SCMRatisSnapshotInfo; import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.hdds.scm.ScmConfig; import org.apache.hadoop.hdds.scm.ScmConfigKeys; @@ -128,7 +122,6 @@ import org.apache.hadoop.util.JvmPauseMonitor; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT_DEFAULT; import org.apache.ratis.grpc.GrpcTlsConfig; -import org.apache.ratis.server.protocol.TermIndex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -202,11 +195,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl private CertificateServer certificateServer; private GrpcTlsConfig grpcTlsConfig; - // SCM HA related - private SCMRatisServer scmRatisServer; - private SCMRatisSnapshotInfo scmRatisSnapshotInfo; - private File scmRatisSnapshotDir; - private JvmPauseMonitor jvmPauseMonitor; private final OzoneConfiguration configuration; private SCMContainerMetrics scmContainerMetrics; @@ -276,10 +264,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl loginAsSCMUser(conf); } - this.scmRatisSnapshotInfo = new SCMRatisSnapshotInfo( - scmStorageConfig.getCurrentDir()); - this.scmRatisSnapshotDir = SCMHAUtils.createSCMRatisDir(conf); - // Creates the SCM DBs or opens them if it exists. // A valid pointer to the store is required by all the other services below. initalizeMetadataStore(conf, configurator); @@ -400,13 +384,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl public static StorageContainerManager createSCM( OzoneConfiguration conf, SCMConfigurator configurator) throws IOException, AuthenticationException { - StorageContainerManager scm = new StorageContainerManager( - conf, configurator); - if (SCMHAUtils.isSCMHAEnabled(conf) && scm.getScmRatisServer() == null) { - SCMRatisServer scmRatisServer = initializeRatisServer(conf, scm); - scm.setScmRatisServer(scmRatisServer); - } - return scm; + return new StorageContainerManager(conf, configurator); } /** @@ -840,10 +818,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl getClientRpcAddress())); } - if (scmRatisServer != null) { - scmRatisServer.start(); - } - scmHAManager.start(); ms = HddsServerUtil @@ -1180,56 +1154,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl return this.clusterMap; } - private static SCMRatisServer initializeRatisServer( - OzoneConfiguration conf, StorageContainerManager scm) throws IOException { - SCMNodeDetails scmNodeDetails = SCMNodeDetails - .initStandAlone(conf); - //TODO enable Ratis group - SCMRatisServer scmRatisServer = SCMRatisServer.newSCMRatisServer( - conf.getObject(SCMRatisServer.SCMRatisServerConfiguration.class), - scm, scmNodeDetails, Collections.EMPTY_LIST, - SCMRatisServer.getSCMRatisDirectory(conf)); - if (scmRatisServer != null) { - LOG.info("SCM Ratis server initialized at port {}", - scmRatisServer.getServerPort()); - } // TODO error handling for scmRatisServer creation failure - return scmRatisServer; - } - - @VisibleForTesting - public SCMRatisServer getScmRatisServer() { - return scmRatisServer; - } - - public void setScmRatisServer(SCMRatisServer scmRatisServer) { - this.scmRatisServer = scmRatisServer; - } - - @VisibleForTesting - public SCMRatisSnapshotInfo getSnapshotInfo() { - return scmRatisSnapshotInfo; - } - - @VisibleForTesting - public long getRatisSnapshotIndex() { - return scmRatisSnapshotInfo.getIndex(); - } - - /** - * Save ratis snapshot to SCM meta store and local disk. - */ - public TermIndex saveRatisSnapshot() throws IOException { - TermIndex snapshotIndex = scmRatisServer.getLastAppliedTermIndex(); - if (scmMetadataStore != null) { - // Flush the SCM state to disk - scmMetadataStore.getStore().flushDB(); - } - - scmRatisSnapshotInfo.saveRatisSnapshotToDisk(snapshotIndex); - - return snapshotIndex; - } - /** * Get the safe mode status of all rules. * diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ratis/SCMStateMachine.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ratis/SCMStateMachine.java index 144380a..9a725a6 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ratis/SCMStateMachine.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ratis/SCMStateMachine.java @@ -73,7 +73,8 @@ public class SCMStateMachine extends BaseStateMachine { this.scmRatisServer = ratisServer; this.scm = ratisServer.getSCM(); - this.snapshotInfo = scm.getSnapshotInfo(); + // TODO: remove the whole file later + this.snapshotInfo = null; updateLastAppliedIndexWithSnaphsotIndex(); ThreadFactory build = new ThreadFactoryBuilder().setDaemon(true) @@ -158,7 +159,8 @@ public class SCMStateMachine extends BaseStateMachine { public long takeSnapshot() throws IOException { LOG.info("Saving Ratis snapshot on the SCM."); if (scm != null) { - return scm.saveRatisSnapshot().getIndex(); + // TODO: remove the whole file later + return 0; } return 0; } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMRatisServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMRatisServer.java deleted file mode 100644 index 8a233ae..0000000 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMRatisServer.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdds.scm.server.ratis; - -import org.apache.hadoop.hdds.HddsConfigKeys; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.scm.ScmConfigKeys; -import org.apache.hadoop.hdds.scm.TestUtils; -import org.apache.hadoop.hdds.scm.ha.SCMNodeDetails; -import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; -import org.apache.hadoop.hdds.scm.server.StorageContainerManager; -import org.apache.hadoop.test.GenericTestUtils; -import org.apache.ratis.protocol.RaftGroupId; -import org.apache.ratis.util.LifeCycle; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.IOException; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Collections; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - -import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; - -/** - * Test class for SCM Ratis Server. - */ -public class TestSCMRatisServer { - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - private OzoneConfiguration conf; - private SCMRatisServer scmRatisServer; - private StorageContainerManager scm; - private String scmId; - private static final long LEADER_ELECTION_TIMEOUT = 500L; - - @Before - public void init() throws Exception { - conf = new OzoneConfiguration(); - scmId = UUID.randomUUID().toString(); - conf.setTimeDuration( - ScmConfigKeys.OZONE_SCM_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY, - LEADER_ELECTION_TIMEOUT, TimeUnit.MILLISECONDS); - conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true); - conf.set(ScmConfigKeys.OZONE_SCM_INTERNAL_SERVICE_ID, "scm-ha-test"); - - // Standalone SCM Ratis server - initSCM(); - scm = TestUtils.getScm(conf); - scm.start(); - scmRatisServer = scm.getScmRatisServer(); - } - - @After - public void shutdown() { - if (scmRatisServer != null) { - scmRatisServer.stop(); - } - if (scm != null) { - scm.stop(); - } - } - - @Test - public void testStartSCMRatisServer() { - Assert.assertEquals("Ratis Server should be in running state", - LifeCycle.State.RUNNING, scmRatisServer.getServerState()); - } - - @Test - public void verifyRaftGroupIdGenerationWithCustomOmServiceId() throws - Exception { - String customScmServiceId = "scmIdCustom123"; - OzoneConfiguration newConf = new OzoneConfiguration(); - String newOmId = UUID.randomUUID().toString(); - String path = GenericTestUtils.getTempPath(newOmId); - Path metaDirPath = Paths.get(path, "scm-meta"); - newConf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDirPath.toString()); - newConf.setTimeDuration( - ScmConfigKeys.OZONE_SCM_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY, - LEADER_ELECTION_TIMEOUT, TimeUnit.MILLISECONDS); - int ratisPort = 9873; - InetSocketAddress rpcAddress = new InetSocketAddress( - InetAddress.getLocalHost(), 0); - SCMNodeDetails nodeDetails = new SCMNodeDetails.Builder() - .setRpcAddress(rpcAddress) - .setRatisPort(ratisPort) - .setSCMNodeId(newOmId) - .setSCMServiceId(customScmServiceId) - .build(); - // Starts a single node Ratis server - scmRatisServer.stop(); - SCMRatisServer newScmRatisServer = SCMRatisServer - .newSCMRatisServer(newConf.getObject(SCMRatisServer - .SCMRatisServerConfiguration.class), scm, nodeDetails, - Collections.emptyList(), - SCMRatisServer.getSCMRatisDirectory(newConf)); - newScmRatisServer.start(); - - UUID uuid = UUID.nameUUIDFromBytes(customScmServiceId.getBytes()); - RaftGroupId raftGroupId = newScmRatisServer.getRaftGroup().getGroupId(); - Assert.assertEquals(uuid, raftGroupId.getUuid()); - Assert.assertEquals(raftGroupId.toByteString().size(), 16); - newScmRatisServer.stop(); - } - - private void initSCM() throws IOException { - String clusterId = UUID.randomUUID().toString(); - scmId = UUID.randomUUID().toString(); - - final String path = folder.newFolder().toString(); - Path scmPath = Paths.get(path, "scm-meta"); - Files.createDirectories(scmPath); - conf.set(OZONE_METADATA_DIRS, scmPath.toString()); - SCMStorageConfig scmStore = new SCMStorageConfig(conf); - scmStore.setClusterId(clusterId); - scmStore.setScmId(scmId); - // writes the version file properties - scmStore.initialize(); - } -} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMStateMachine.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMStateMachine.java deleted file mode 100644 index 0eddbde..0000000 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMStateMachine.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hadoop.hdds.scm.server.ratis; - -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.scm.ScmConfigKeys; -import org.apache.hadoop.hdds.scm.TestUtils; -import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; -import org.apache.hadoop.hdds.scm.server.StorageContainerManager; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.UUID; - -import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; - -/** - * Test class for SCMStateMachine. - */ -public class TestSCMStateMachine { - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - private SCMStateMachine scmStateMachine; - private StorageContainerManager scm; - private SCMRatisServer scmRatisServer; - private OzoneConfiguration conf; - private String scmId; - @Before - public void init() throws Exception { - conf = new OzoneConfiguration(); - conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true); - conf.set(ScmConfigKeys.OZONE_SCM_INTERNAL_SERVICE_ID, "scm-ha-test"); - scmId = UUID.randomUUID().toString(); - - initSCM(); - scm = TestUtils.getScm(conf); - scm.start(); - scmRatisServer = scm.getScmRatisServer(); - scmStateMachine = scm.getScmRatisServer().getScmStateMachine(); - } - - @Test - public void testSCMUpdatedAppliedIndex(){ - // State machine should start with 0 term and 0 index. - scmStateMachine.notifyIndexUpdate(0, 0); - Assert.assertEquals(0, - scmStateMachine.getLastAppliedTermIndex().getTerm()); - Assert.assertEquals(0, - scmStateMachine.getLastAppliedTermIndex().getIndex()); - - // If only the transactionMap is updated, index should stay 0. - scmStateMachine.addApplyTransactionTermIndex(0, 1); - Assert.assertEquals(0L, - scmStateMachine.getLastAppliedTermIndex().getTerm()); - Assert.assertEquals(0L, - scmStateMachine.getLastAppliedTermIndex().getIndex()); - - // After the index update is notified, the index should increase. - scmStateMachine.notifyIndexUpdate(0, 1); - Assert.assertEquals(0L, - scmStateMachine.getLastAppliedTermIndex().getTerm()); - Assert.assertEquals(1L, - scmStateMachine.getLastAppliedTermIndex().getIndex()); - - // Only do a notifyIndexUpdate can also increase the index. - scmStateMachine.notifyIndexUpdate(0, 2); - Assert.assertEquals(0L, - scmStateMachine.getLastAppliedTermIndex().getTerm()); - Assert.assertEquals(2L, - scmStateMachine.getLastAppliedTermIndex().getIndex()); - - // If a larger index is notified, the index should not be updated. - scmStateMachine.notifyIndexUpdate(0, 5); - Assert.assertEquals(0L, - scmStateMachine.getLastAppliedTermIndex().getTerm()); - Assert.assertEquals(2L, - scmStateMachine.getLastAppliedTermIndex().getIndex()); - } - - private void initSCM() throws IOException { - String clusterId = UUID.randomUUID().toString(); - final String path = folder.newFolder().toString(); - Path scmPath = Paths.get(path, "scm-meta"); - Files.createDirectories(scmPath); - conf.set(OZONE_METADATA_DIRS, scmPath.toString()); - SCMStorageConfig scmStore = new SCMStorageConfig(conf); - scmStore.setClusterId(clusterId); - scmStore.setScmId(scmId); - // writes the version file properties - scmStore.initialize(); - } - - @After - public void cleanup() { - scm.stop(); - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org