This is an automated email from the ASF dual-hosted git repository. avijayan pushed a commit to branch HDDS-3698-upgrade in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 08d8d36c66f5c10289f56e41d2be481c39b406ae Author: avijayanhwx <14299376+avijayan...@users.noreply.github.com> AuthorDate: Tue Sep 29 09:41:43 2020 -0700 HDDS-4227. Implement a 'Prepare For Upgrade' step in OM that applies all committed Ratis transactions. (#1430) --- hadoop-hdds/common/pom.xml | 8 ++ .../hadoop/hdds/ratis/RatisUpgradeUtils.java | 96 +++++++++++++++++++++ .../hadoop/hdds/ratis/TestRatisUpgradeUtils.java | 97 ++++++++++++++++++++++ .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 4 + .../apache/hadoop/ozone/om/OMStarterInterface.java | 2 + .../org/apache/hadoop/ozone/om/OzoneManager.java | 64 ++++++++++++-- .../hadoop/ozone/om/OzoneManagerStarter.java | 38 +++++++++ .../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 2 +- .../ozone/om/ratis/OzoneManagerRatisServer.java | 4 + .../hadoop/ozone/om/TestOzoneManagerStarter.java | 11 +++ 10 files changed, 319 insertions(+), 7 deletions(-) diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml index 8525976..4a17336 100644 --- a/hadoop-hdds/common/pom.xml +++ b/hadoop-hdds/common/pom.xml @@ -193,6 +193,14 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> <artifactId>hamcrest-all</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>com.codahale.metrics</groupId> + <artifactId>metrics-core</artifactId> + <version>3.0.2</version> + <scope>test</scope> + <!-- Needed for mocking RaftServerImpl --> + </dependency> + </dependencies> <build> diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisUpgradeUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisUpgradeUtils.java new file mode 100644 index 0000000..796668d --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisUpgradeUtils.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.hdds.ratis; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +import org.apache.ratis.protocol.RaftGroup; +import org.apache.ratis.server.impl.RaftServerImpl; +import org.apache.ratis.server.impl.RaftServerProxy; +import org.apache.ratis.statemachine.StateMachine; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Ratis utility functions. + */ +public final class RatisUpgradeUtils { + + private RatisUpgradeUtils() { + } + + private static final Logger LOG = + LoggerFactory.getLogger(RatisUpgradeUtils.class); + + /** + * Flush all committed transactions in a given Raft Server for a given group. + * @param stateMachine state machine to use + * @param raftGroup raft group + * @param server Raft server proxy instance. + * @param maxTimeToWaitSeconds Max time to wait before declaring failure. + * @throws InterruptedException when interrupted + * @throws IOException on error while waiting + */ + public static void waitForAllTxnsApplied( + StateMachine stateMachine, + RaftGroup raftGroup, + RaftServerProxy server, + long maxTimeToWaitSeconds, + long timeBetweenRetryInSeconds) + throws InterruptedException, IOException { + + long intervalTime = TimeUnit.SECONDS.toMillis(timeBetweenRetryInSeconds); + long endTime = System.currentTimeMillis() + + TimeUnit.SECONDS.toMillis(maxTimeToWaitSeconds); + boolean success = false; + while (System.currentTimeMillis() < endTime) { + success = checkIfAllTransactionsApplied(stateMachine, server, raftGroup); + if (success) { + break; + } + Thread.sleep(intervalTime); + } + + if (!success) { + throw new IOException(String.format("After waiting for %d seconds, " + + "State Machine has not applied all the transactions.", + maxTimeToWaitSeconds)); + } + + long snapshotIndex = stateMachine.takeSnapshot(); + if (snapshotIndex != stateMachine.getLastAppliedTermIndex().getIndex()) { + throw new IOException("Index from Snapshot does not match last applied " + + "Index"); + } + } + + private static boolean checkIfAllTransactionsApplied( + StateMachine stateMachine, + RaftServerProxy serverProxy, + RaftGroup raftGroup) throws IOException { + LOG.info("Checking for pending transactions to be applied."); + RaftServerImpl impl = serverProxy.getImpl(raftGroup.getGroupId()); + long lastCommittedIndex = impl.getState().getLog().getLastCommittedIndex(); + long appliedIndex = stateMachine.getLastAppliedTermIndex().getIndex(); + LOG.info("lastCommittedIndex = {}, appliedIndex = {}", + lastCommittedIndex, appliedIndex); + return (lastCommittedIndex == appliedIndex); + } + +} diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/ratis/TestRatisUpgradeUtils.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/ratis/TestRatisUpgradeUtils.java new file mode 100644 index 0000000..078bbb5 --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/ratis/TestRatisUpgradeUtils.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.hdds.ratis; + +import static org.apache.hadoop.hdds.ratis.RatisUpgradeUtils.waitForAllTxnsApplied; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; + +import org.apache.hadoop.test.LambdaTestUtils; +import org.apache.ratis.protocol.RaftGroup; +import org.apache.ratis.server.impl.RaftServerImpl; +import org.apache.ratis.server.impl.RaftServerProxy; +import org.apache.ratis.server.impl.ServerState; +import org.apache.ratis.server.protocol.TermIndex; +import org.apache.ratis.server.raftlog.RaftLog; +import org.apache.ratis.statemachine.StateMachine; +import org.junit.Test; + +/** + * Testing util methods in TestRatisUpgradeUtils. + */ +public class TestRatisUpgradeUtils { + + @Test + public void testWaitForAllTxnsApplied() throws IOException, + InterruptedException { + + StateMachine stateMachine = mock(StateMachine.class); + RaftGroup raftGroup = RaftGroup.emptyGroup(); + RaftServerProxy raftServerProxy = mock(RaftServerProxy.class); + RaftServerImpl raftServer = mock(RaftServerImpl.class); + ServerState serverState = mock(ServerState.class); + RaftLog raftLog = mock(RaftLog.class); + + when(raftServerProxy.getImpl( + raftGroup.getGroupId())).thenReturn(raftServer); + when(raftServer.getState()).thenReturn(serverState); + when(serverState.getLog()).thenReturn(raftLog); + when(raftLog.getLastCommittedIndex()).thenReturn(1L); + + TermIndex termIndex = mock(TermIndex.class); + when(termIndex.getIndex()).thenReturn(0L).thenReturn(0L).thenReturn(1L); + when(stateMachine.getLastAppliedTermIndex()).thenReturn(termIndex); + when(stateMachine.takeSnapshot()).thenReturn(1L); + + waitForAllTxnsApplied(stateMachine, raftGroup, raftServerProxy, 10, 2); + verify(stateMachine.getLastAppliedTermIndex(), + times(4)); // 3 checks + 1 after snapshot + } + + @Test + public void testWaitForAllTxnsAppliedTimeOut() throws Exception { + + StateMachine stateMachine = mock(StateMachine.class); + RaftGroup raftGroup = RaftGroup.emptyGroup(); + RaftServerProxy raftServerProxy = mock(RaftServerProxy.class); + RaftServerImpl raftServer = mock(RaftServerImpl.class); + ServerState serverState = mock(ServerState.class); + RaftLog raftLog = mock(RaftLog.class); + + when(raftServerProxy.getImpl( + raftGroup.getGroupId())).thenReturn(raftServer); + when(raftServer.getState()).thenReturn(serverState); + when(serverState.getLog()).thenReturn(raftLog); + when(raftLog.getLastCommittedIndex()).thenReturn(1L); + + TermIndex termIndex = mock(TermIndex.class); + when(termIndex.getIndex()).thenReturn(0L); + when(stateMachine.getLastAppliedTermIndex()).thenReturn(termIndex); + when(stateMachine.takeSnapshot()).thenReturn(1L); + + LambdaTestUtils.intercept(IOException.class, "State Machine has not " + + "applied all the transactions", () -> + waitForAllTxnsApplied(stateMachine, raftGroup, raftServerProxy, + 10, 2)); + } + +} \ No newline at end of file diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index f16679a..fba6dcc 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -246,4 +246,8 @@ public final class OMConfigKeys { "ozone.om.enable.filesystem.paths"; public static final boolean OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT = false; + + public static final long OZONE_OM_MAX_TIME_TO_WAIT_FLUSH_TXNS = + TimeUnit.MINUTES.toSeconds(5); + public static final long OZONE_OM_FLUSH_TXNS_RETRY_INTERVAL_SECONDS = 5L; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStarterInterface.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStarterInterface.java index f632ad1..14252a7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStarterInterface.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStarterInterface.java @@ -30,4 +30,6 @@ public interface OMStarterInterface { AuthenticationException; boolean init(OzoneConfiguration conf) throws IOException, AuthenticationException; + boolean prepareForUpgrade(OzoneConfiguration conf) throws IOException, + AuthenticationException; } \ No newline at end of file diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index b7a6b4f..3a2555a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -193,6 +193,7 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED_DEFAULT; import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForBlockClients; import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients; +import static org.apache.hadoop.hdds.ratis.RatisUpgradeUtils.waitForAllTxnsApplied; import static org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest.getEncodedString; import static org.apache.hadoop.hdds.server.ServerUtils.getRemoteUserName; import static org.apache.hadoop.hdds.server.ServerUtils.updateRPCListenAddress; @@ -215,10 +216,12 @@ import static org.apache.hadoop.ozone.OzoneConsts.RPC_PORT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_FLUSH_TXNS_RETRY_INTERVAL_SECONDS; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HANDLER_COUNT_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HANDLER_COUNT_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_MAX_TIME_TO_WAIT_FLUSH_TXNS; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_USER_MAX_VOLUME; @@ -234,6 +237,7 @@ import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos. import org.apache.hadoop.util.Time; import org.apache.ratis.proto.RaftProtos.RaftPeerRole; +import org.apache.ratis.server.impl.RaftServerProxy; import org.apache.ratis.server.protocol.TermIndex; import org.apache.ratis.util.ExitUtils; import org.apache.ratis.util.FileUtils; @@ -327,20 +331,22 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl private final boolean useRatisForReplication; private boolean isNativeAuthorizerEnabled; + private boolean prepareForUpgrade; private ExitManager exitManager; private enum State { INITIALIZED, RUNNING, + PREPARING_FOR_UPGRADE, STOPPED } // Used in MiniOzoneCluster testing private State omState; - private OzoneManager(OzoneConfiguration conf) throws IOException, - AuthenticationException { + private OzoneManager(OzoneConfiguration conf, boolean forUpgrade) + throws IOException, AuthenticationException { super(OzoneVersionInfo.OZONE_VERSION_INFO); Preconditions.checkNotNull(conf); configuration = conf; @@ -486,6 +492,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl }; ShutdownHookManager.get().addShutdownHook(shutdownHook, SHUTDOWN_HOOK_PRIORITY); + this.prepareForUpgrade = forUpgrade; omState = State.INITIALIZED; } @@ -919,7 +926,12 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl */ public static OzoneManager createOm(OzoneConfiguration conf) throws IOException, AuthenticationException { - return new OzoneManager(conf); + return new OzoneManager(conf, false); + } + + public static OzoneManager createOmUpgradeMode(OzoneConfiguration conf) + throws IOException, AuthenticationException { + return new OzoneManager(conf, true); } /** @@ -995,6 +1007,39 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl } } + public boolean applyAllPendingTransactions() + throws InterruptedException, IOException { + + if (!isRatisEnabled) { + LOG.info("Ratis not enabled. Nothing to do."); + return true; + } + + waitForAllTxnsApplied(omRatisServer.getOmStateMachine(), + omRatisServer.getRaftGroup(), + (RaftServerProxy) omRatisServer.getServer(), + OZONE_OM_MAX_TIME_TO_WAIT_FLUSH_TXNS, + OZONE_OM_FLUSH_TXNS_RETRY_INTERVAL_SECONDS); + + long appliedIndexFromRatis = + omRatisServer.getOmStateMachine().getLastAppliedTermIndex().getIndex(); + OMTransactionInfo omTransactionInfo = + OMTransactionInfo.readTransactionInfo(metadataManager); + long index = omTransactionInfo.getTermIndex().getIndex(); + if (index != appliedIndexFromRatis) { + throw new IllegalStateException( + String.format("Cannot prepare OM for Upgrade " + + "since transaction info table index %d does not match ratis %s", + index, appliedIndexFromRatis)); + } + + LOG.info("OM has been prepared for upgrade. All transactions " + + "upto {} have been flushed to the state machine, " + + "and a snapshot has been taken.", + omRatisServer.getOmStateMachine().getLastAppliedTermIndex().getIndex()); + return true; + } + /** * Initializes secure OzoneManager. */ @@ -1180,15 +1225,22 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl // Allow OM to start as Http Server failure is not fatal. LOG.error("OM HttpServer failed to start.", ex); } - omRpcServer.start(); - isOmRpcServerRunning = true; + if (!prepareForUpgrade) { + omRpcServer.start(); + isOmRpcServerRunning = true; + } registerMXBean(); startJVMPauseMonitor(); setStartTime(); - omState = State.RUNNING; + if (!prepareForUpgrade) { + omState = State.RUNNING; + } else { + omState = State.PREPARING_FOR_UPGRADE; + LOG.info("Started OM services in upgrade mode."); + } } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java index 6dc4aea..936980b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java @@ -98,6 +98,28 @@ public class OzoneManagerStarter extends GenericCli { } } + + /** + * This function implements a sub-command to allow the OM to be + * "prepared for upgrade". + */ + @CommandLine.Command(name = "--prepareForUpgrade", + aliases = {"--prepareForDowngrade", "--flushTransactions"}, + customSynopsis = "ozone om [global options] --prepareForUpgrade", + hidden = false, + description = "Prepare the OM for upgrade/downgrade. (Flush Raft log " + + "transactions.)", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class) + public void prepareOmForUpgrade() throws Exception { + commonInit(); + boolean result = receiver.prepareForUpgrade(conf); + if (!result) { + throw new Exception("Prepare OM For Upgrade failed."); + } + System.exit(0); + } + /** * This function should be called by each command to ensure the configuration * is set and print the startup banner message. @@ -130,6 +152,22 @@ public class OzoneManagerStarter extends GenericCli { AuthenticationException { return OzoneManager.omInit(conf); } + + public boolean prepareForUpgrade(OzoneConfiguration conf) + throws IOException, AuthenticationException { + try (OzoneManager om = OzoneManager.createOmUpgradeMode(conf)) { + om.start(); + boolean success = false; + try { + LOG.info("Preparing OM for upgrade."); + success = om.applyAllPendingTransactions(); + } catch (InterruptedException e) { + LOG.error("Error preparing OM for upgrade.", e); + Thread.currentThread().interrupt(); + } + return success; + } + } } } \ No newline at end of file diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java index 68d359e..dcf7984 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java @@ -319,7 +319,7 @@ public final class OzoneManagerDoubleBuffer { if (LOG.isDebugEnabled()) { LOG.debug("Sync Iteration {} flushed transactions in this " + - "iteration{}", flushIterations.get(), + "iteration {}", flushIterations.get(), flushedTransactionsSize); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java index 4b8c11a..c5868f1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java @@ -698,4 +698,8 @@ public final class OzoneManagerRatisServer { public TermIndex getLastAppliedTermIndex() { return omStateMachine.getLastAppliedTermIndex(); } + + public RaftServer getServer() { + return server; + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerStarter.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerStarter.java index 8028169..c137991 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerStarter.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerStarter.java @@ -133,6 +133,8 @@ public class TestOzoneManagerStarter { private boolean initStatus = true; private boolean throwOnStart = false; private boolean throwOnInit = false; + private boolean prepareUpgradeCalled = false; + private boolean throwOnPrepareUpgrade = false; public void start(OzoneConfiguration conf) throws IOException, AuthenticationException { @@ -150,5 +152,14 @@ public class TestOzoneManagerStarter { } return initStatus; } + + public boolean prepareForUpgrade(OzoneConfiguration conf) + throws IOException, AuthenticationException { + prepareUpgradeCalled = true; + if (throwOnPrepareUpgrade) { + throw new IOException("Simulated Exception"); + } + return true; + } } } \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org