HDFS-13805. Journal Nodes should allow to format non-empty directories with -force option. Contributed by Surendra Singh Lilhore.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96c4575d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96c4575d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96c4575d Branch: refs/heads/HDFS-12943 Commit: 96c4575d7373079becfa3e3db29ba98e6fb86388 Parents: ca29fb7 Author: Surendra Singh Lilhore <surendralilh...@apache.org> Authored: Fri Aug 24 08:14:57 2018 +0530 Committer: Surendra Singh Lilhore <surendralilh...@apache.org> Committed: Fri Aug 24 08:14:57 2018 +0530 ---------------------------------------------------------------------- .../hadoop/hdfs/qjournal/client/AsyncLogger.java | 3 ++- .../hdfs/qjournal/client/AsyncLoggerSet.java | 4 ++-- .../hdfs/qjournal/client/IPCLoggerChannel.java | 5 +++-- .../hdfs/qjournal/client/QuorumJournalManager.java | 4 ++-- .../hdfs/qjournal/protocol/QJournalProtocol.java | 2 +- .../QJournalProtocolServerSideTranslatorPB.java | 2 +- .../protocolPB/QJournalProtocolTranslatorPB.java | 6 ++++-- .../hadoop/hdfs/qjournal/server/JNStorage.java | 4 ++-- .../hadoop/hdfs/qjournal/server/Journal.java | 6 +++--- .../hdfs/qjournal/server/JournalNodeRpcServer.java | 5 +++-- .../hdfs/server/namenode/BackupJournalManager.java | 2 +- .../hadoop/hdfs/server/namenode/FSEditLog.java | 5 +++-- .../hadoop/hdfs/server/namenode/FSImage.java | 5 +++-- .../hadoop/hdfs/server/namenode/FSNamesystem.java | 4 ++-- .../hdfs/server/namenode/FileJournalManager.java | 2 +- .../hdfs/server/namenode/JournalManager.java | 2 +- .../hadoop/hdfs/server/namenode/JournalSet.java | 2 +- .../hadoop/hdfs/server/namenode/NameNode.java | 4 ++-- .../src/main/proto/QJournalProtocol.proto | 1 + .../hdfs/qjournal/client/TestEpochsAreUnique.java | 2 +- .../hdfs/qjournal/client/TestQJMWithFaults.java | 6 +++--- .../qjournal/client/TestQuorumJournalManager.java | 4 ++-- .../client/TestQuorumJournalManagerUnit.java | 4 +++- .../hadoop/hdfs/qjournal/server/TestJournal.java | 17 ++++++++++++++--- .../hdfs/qjournal/server/TestJournalNode.java | 4 ++-- .../qjournal/server/TestJournalNodeMXBean.java | 2 +- .../hdfs/qjournal/server/TestJournalNodeSync.java | 2 +- .../server/namenode/TestGenericJournalConf.java | 2 +- .../tools/contract/AbstractContractDistCpTest.java | 2 +- 29 files changed, 67 insertions(+), 46 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java index d2b48cc..2633723 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java @@ -89,8 +89,9 @@ interface AsyncLogger { /** * Format the log directory. * @param nsInfo the namespace info to format with + * @param force the force option to format */ - public ListenableFuture<Void> format(NamespaceInfo nsInfo); + public ListenableFuture<Void> format(NamespaceInfo nsInfo, boolean force); /** * @return whether or not the remote node has any valid data. http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java index d46c2cf..b52e312 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java @@ -299,12 +299,12 @@ class AsyncLoggerSet { return QuorumCall.create(calls); } - QuorumCall<AsyncLogger,Void> format(NamespaceInfo nsInfo) { + QuorumCall<AsyncLogger, Void> format(NamespaceInfo nsInfo, boolean force) { Map<AsyncLogger, ListenableFuture<Void>> calls = Maps.newHashMap(); for (AsyncLogger logger : loggers) { ListenableFuture<Void> future = - logger.format(nsInfo); + logger.format(nsInfo, force); calls.put(logger, future); } return QuorumCall.create(calls); http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java index 3036735..4fca1bb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java @@ -502,11 +502,12 @@ public class IPCLoggerChannel implements AsyncLogger { } @Override - public ListenableFuture<Void> format(final NamespaceInfo nsInfo) { + public ListenableFuture<Void> format(final NamespaceInfo nsInfo, + final boolean force) { return singleThreadExecutor.submit(new Callable<Void>() { @Override public Void call() throws Exception { - getProxy().format(journalId, nameServiceId, nsInfo); + getProxy().format(journalId, nameServiceId, nsInfo, force); return null; } }); http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java index 4faaa98..bd45292 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java @@ -220,8 +220,8 @@ public class QuorumJournalManager implements JournalManager { } @Override - public void format(NamespaceInfo nsInfo) throws IOException { - QuorumCall<AsyncLogger,Void> call = loggers.format(nsInfo); + public void format(NamespaceInfo nsInfo, boolean force) throws IOException { + QuorumCall<AsyncLogger, Void> call = loggers.format(nsInfo, force); try { call.waitFor(loggers.size(), loggers.size(), 0, timeoutMs, "format"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java index 5558bd5..8dad261 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java @@ -68,7 +68,7 @@ public interface QJournalProtocol { * Format the underlying storage for the given namespace. */ void format(String journalId, String nameServiceId, - NamespaceInfo nsInfo) throws IOException; + NamespaceInfo nsInfo, boolean force) throws IOException; /** * Begin a new epoch. See the HDFS-3077 design doc for details. http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java index 865d296..2ad19da 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java @@ -147,7 +147,7 @@ public class QJournalProtocolServerSideTranslatorPB implements QJournalProtocolP try { impl.format(request.getJid().getIdentifier(), request.hasNameServiceId() ? request.getNameServiceId() : null, - PBHelper.convert(request.getNsInfo())); + PBHelper.convert(request.getNsInfo()), request.getForce()); return FormatResponseProto.getDefaultInstance(); } catch (IOException ioe) { throw new ServiceException(ioe); http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java index d7cd7b5..42d35f5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java @@ -136,11 +136,13 @@ public class QJournalProtocolTranslatorPB implements ProtocolMetaInterface, @Override public void format(String jid, String nameServiceId, - NamespaceInfo nsInfo) throws IOException { + NamespaceInfo nsInfo, + boolean force) throws IOException { try { FormatRequestProto.Builder req = FormatRequestProto.newBuilder() .setJid(convertJournalId(jid)) - .setNsInfo(PBHelper.convert(nsInfo)); + .setNsInfo(PBHelper.convert(nsInfo)) + .setForce(force); if(nameServiceId != null) { req.setNameServiceId(nameServiceId); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java index 6bf4903..612fd3d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java @@ -204,10 +204,10 @@ class JNStorage extends Storage { } } - void format(NamespaceInfo nsInfo) throws IOException { + void format(NamespaceInfo nsInfo, boolean force) throws IOException { unlockAll(); try { - sd.analyzeStorage(StartupOption.FORMAT, this, true); + sd.analyzeStorage(StartupOption.FORMAT, this, !force); } finally { sd.unlock(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java index 8f25d26..7e88afa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java @@ -227,13 +227,13 @@ public class Journal implements Closeable { /** * Format the local storage with the given namespace. */ - void format(NamespaceInfo nsInfo) throws IOException { + void format(NamespaceInfo nsInfo, boolean force) throws IOException { Preconditions.checkState(nsInfo.getNamespaceID() != 0, "can't format with uninitialized namespace info: %s", nsInfo); LOG.info("Formatting journal id : " + journalId + " with namespace info: " + - nsInfo); - storage.format(nsInfo); + nsInfo + " and force: " + force); + storage.format(nsInfo, force); refreshCachedData(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java index b1a3c96..0f11026 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java @@ -176,9 +176,10 @@ public class JournalNodeRpcServer implements QJournalProtocol, @Override public void format(String journalId, String nameServiceId, - NamespaceInfo nsInfo) + NamespaceInfo nsInfo, + boolean force) throws IOException { - jn.getOrCreateJournal(journalId, nameServiceId).format(nsInfo); + jn.getOrCreateJournal(journalId, nameServiceId).format(nsInfo, force); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java index e1ddfb9..eac91bf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java @@ -42,7 +42,7 @@ class BackupJournalManager implements JournalManager { } @Override - public void format(NamespaceInfo nsInfo) { + public void format(NamespaceInfo nsInfo, boolean force) { // format() should only get called at startup, before any BNs // can register with the NN. throw new UnsupportedOperationException( http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java index d6fb212..547ad57 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java @@ -417,13 +417,14 @@ public class FSEditLog implements LogsPurgeable { * File-based journals are skipped, since they are formatted by the * Storage format code. */ - synchronized void formatNonFileJournals(NamespaceInfo nsInfo) throws IOException { + synchronized void formatNonFileJournals(NamespaceInfo nsInfo, boolean force) + throws IOException { Preconditions.checkState(state == State.BETWEEN_LOG_SEGMENTS, "Bad state: %s", state); for (JournalManager jm : journalSet.getJournalManagers()) { if (!(jm instanceof FileJournalManager)) { - jm.format(nsInfo); + jm.format(nsInfo, force); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index 5cfc017..6d107be 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -160,7 +160,8 @@ public class FSImage implements Closeable { archivalManager = new NNStorageRetentionManager(conf, storage, editLog); } - void format(FSNamesystem fsn, String clusterId) throws IOException { + void format(FSNamesystem fsn, String clusterId, boolean force) + throws IOException { long fileCount = fsn.getFilesTotal(); // Expect 1 file, which is the root inode Preconditions.checkState(fileCount == 1, @@ -171,7 +172,7 @@ public class FSImage implements Closeable { ns.clusterID = clusterId; storage.format(ns); - editLog.formatNonFileJournals(ns); + editLog.formatNonFileJournals(ns, force); saveFSImageInAllDirs(fsn, 0); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index b0fb26c..06bf008 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -1078,8 +1078,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, // format before starting up if requested if (startOpt == StartupOption.FORMAT) { - - fsImage.format(this, fsImage.getStorage().determineClusterId());// reuse current id + // reuse current id + fsImage.format(this, fsImage.getStorage().determineClusterId(), false); startOpt = StartupOption.REGULAR; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java index c71c09a..185ad73 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java @@ -100,7 +100,7 @@ public class FileJournalManager implements JournalManager { public void close() throws IOException {} @Override - public void format(NamespaceInfo ns) throws IOException { + public void format(NamespaceInfo ns, boolean force) throws IOException { // Formatting file journals is done by the StorageDirectory // format code, since they may share their directory with // checkpoints, etc. http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java index ae1bc3b..d6d2094 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java @@ -43,7 +43,7 @@ public interface JournalManager extends Closeable, FormatConfirmable, * Format the underlying storage, removing any previously * stored data. */ - void format(NamespaceInfo ns) throws IOException; + void format(NamespaceInfo ns, boolean force) throws IOException; /** * Begin writing to a new segment of the log stream, which starts at http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java index e7f2adb..868df01 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java @@ -188,7 +188,7 @@ public class JournalSet implements JournalManager { } @Override - public void format(NamespaceInfo nsInfo) throws IOException { + public void format(NamespaceInfo nsInfo, boolean force) throws IOException { // The operation is done by FSEditLog itself throw new UnsupportedOperationException(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 7f78d2f..a8034da 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -1159,7 +1159,7 @@ public class NameNode extends ReconfigurableBase implements return true; // aborted } - fsImage.format(fsn, clusterId); + fsImage.format(fsn, clusterId, force); } catch (IOException ioe) { LOG.warn("Encountered exception during format: ", ioe); fsImage.close(); @@ -1262,7 +1262,7 @@ public class NameNode extends ReconfigurableBase implements // actually want to save a checkpoint - just prime the dirs with // the existing namespace info newSharedStorage.format(nsInfo); - sharedEditsImage.getEditLog().formatNonFileJournals(nsInfo); + sharedEditsImage.getEditLog().formatNonFileJournals(nsInfo, force); // Need to make sure the edit log segments are in good shape to initialize // the shared edits dir. http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto index a37c723..625966f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto @@ -242,6 +242,7 @@ message FormatRequestProto { required JournalIdProto jid = 1; required NamespaceInfoProto nsInfo = 2; optional string nameServiceId = 3; + optional bool force = 4 [ default = false ]; } message FormatResponseProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestEpochsAreUnique.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestEpochsAreUnique.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestEpochsAreUnique.java index 5101a41..0fc1429 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestEpochsAreUnique.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestEpochsAreUnique.java @@ -56,7 +56,7 @@ public class TestEpochsAreUnique { QuorumJournalManager qjm = new QuorumJournalManager( conf, uri, FAKE_NSINFO); try { - qjm.format(FAKE_NSINFO); + qjm.format(FAKE_NSINFO, false); } finally { qjm.close(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java index 6ad43f5..40f213e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java @@ -105,7 +105,7 @@ public class TestQJMWithFaults { long ret; try { qjm = createInjectableQJM(cluster); - qjm.format(FAKE_NSINFO); + qjm.format(FAKE_NSINFO, false); doWorkload(cluster, qjm); SortedSet<Integer> ipcCounts = Sets.newTreeSet(); @@ -156,7 +156,7 @@ public class TestQJMWithFaults { QuorumJournalManager qjm = null; try { qjm = createInjectableQJM(cluster); - qjm.format(FAKE_NSINFO); + qjm.format(FAKE_NSINFO, false); List<AsyncLogger> loggers = qjm.getLoggerSetForTests().getLoggersForTests(); failIpcNumber(loggers.get(0), failA); failIpcNumber(loggers.get(1), failB); @@ -240,7 +240,7 @@ public class TestQJMWithFaults { // Format the cluster using a non-faulty QJM. QuorumJournalManager qjmForInitialFormat = createInjectableQJM(cluster); - qjmForInitialFormat.format(FAKE_NSINFO); + qjmForInitialFormat.format(FAKE_NSINFO, false); qjmForInitialFormat.close(); try { http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java index 69856ae..00bec22 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java @@ -100,7 +100,7 @@ public class TestQuorumJournalManager { qjm = createSpyingQJM(); spies = qjm.getLoggerSetForTests().getLoggersForTests(); - qjm.format(QJMTestUtil.FAKE_NSINFO); + qjm.format(QJMTestUtil.FAKE_NSINFO, false); qjm.recoverUnfinalizedSegments(); assertEquals(1, qjm.getLoggerSetForTests().getEpoch()); } @@ -149,7 +149,7 @@ public class TestQuorumJournalManager { QuorumJournalManager qjm = closeLater(new QuorumJournalManager( conf, cluster.getQuorumJournalURI("testFormat-jid"), FAKE_NSINFO)); assertFalse(qjm.hasSomeData()); - qjm.format(FAKE_NSINFO); + qjm.format(FAKE_NSINFO, false); assertTrue(qjm.hasSomeData()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManagerUnit.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManagerUnit.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManagerUnit.java index 75dcf2f..9e1e3bb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManagerUnit.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManagerUnit.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.qjournal.client; import static org.junit.Assert.fail; import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.eq; import java.io.IOException; @@ -89,7 +90,8 @@ public class TestQuorumJournalManagerUnit { NewEpochResponseProto.newBuilder().build() ).when(logger).newEpoch(Mockito.anyLong()); - futureReturns(null).when(logger).format(Mockito.<NamespaceInfo>any()); + futureReturns(null).when(logger).format(Mockito.<NamespaceInfo>any(), + anyBoolean()); } qjm.recoverUnfinalizedSegments(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournal.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournal.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournal.java index b71d694..b8d2652 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournal.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournal.java @@ -73,7 +73,7 @@ public class TestJournal { conf = new Configuration(); journal = new Journal(conf, TEST_LOG_DIR, JID, StartupOption.REGULAR, mockErrorReporter); - journal.format(FAKE_NSINFO); + journal.format(FAKE_NSINFO, false); } @After @@ -207,7 +207,7 @@ public class TestJournal { // Clear the storage directory before reformatting it journal.getStorage().getJournalManager() .getStorageDirectory().clearDirectory(); - journal.format(FAKE_NSINFO_2); + journal.format(FAKE_NSINFO_2, false); assertEquals(0, journal.getLastPromisedEpoch()); assertEquals(0, journal.getLastWriterEpoch()); @@ -425,7 +425,7 @@ public class TestJournal { try { // Format again here and to format the non-empty directories in // journal node. - journal.format(FAKE_NSINFO); + journal.format(FAKE_NSINFO, false); fail("Did not fail to format non-empty directories in journal node."); } catch (IOException ioe) { GenericTestUtils.assertExceptionContains( @@ -434,4 +434,15 @@ public class TestJournal { } } + @Test + public void testFormatNonEmptyStorageDirectoriesWhenforceOptionIsTrue() + throws Exception { + try { + // Format again here and to format the non-empty directories in + // journal node. + journal.format(FAKE_NSINFO, true); + } catch (IOException ioe) { + fail("Format should be success with force option."); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java index 8d58792..4cc5968 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java @@ -159,11 +159,11 @@ public class TestJournalNode { HdfsServerConstants.StartupOption.REGULAR); NamespaceInfo fakeNameSpaceInfo = new NamespaceInfo( 12345, "mycluster", "my-bp"+nsId, 0L); - journal.format(fakeNameSpaceInfo); + journal.format(fakeNameSpaceInfo, false); } } else { journal = jn.getOrCreateJournal(journalId); - journal.format(FAKE_NSINFO); + journal.format(FAKE_NSINFO, false); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeMXBean.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeMXBean.java index 1de37a4..7550c4e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeMXBean.java @@ -82,7 +82,7 @@ public class TestJournalNodeMXBean { // format the journal ns1 final NamespaceInfo FAKE_NSINFO = new NamespaceInfo(12345, "mycluster", "my-bp", 0L); - jn.getOrCreateJournal(NAMESERVICE).format(FAKE_NSINFO); + jn.getOrCreateJournal(NAMESERVICE).format(FAKE_NSINFO, false); // check again after format // getJournalsStatus http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeSync.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeSync.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeSync.java index 8de9641..c23604b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeSync.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeSync.java @@ -341,7 +341,7 @@ public class TestJournalNodeSync { } // Format the JN - journal1.format(nsInfo); + journal1.format(nsInfo, false); // Roll some more edits for (int i = 4; i < 10; i++) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java index 020ecb5..edcf9e1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java @@ -155,7 +155,7 @@ public class TestGenericJournalConf { } @Override - public void format(NamespaceInfo nsInfo) throws IOException { + public void format(NamespaceInfo nsInfo, boolean force) throws IOException { formatCalled = true; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c4575d/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java index 5c74430..0757a66 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java @@ -249,7 +249,7 @@ public abstract class AbstractContractDistCpTest Counter c = job.getCounters().findCounter(counter); long value = c.getValue(); String description = - String.format("%s value %s", c.getDisplayName(), value); + String.format("%s value %s", c.getDisplayName(), value, false); if (min >= 0) { assertTrue(description + " too below minimum " + min, --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org