[10/54] [abbrv] hadoop git commit: HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by Ajay Kumar.

2017-10-20 Thread vrushali 
HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by 
Ajay Kumar.

(cherry picked from commit b6942cbe9b8c9469e8c2b64c3268d671f5a43e75)
(cherry picked from commit 5897095d539be086ed37df011f024e37eb37b0cd)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bc3ca4c1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bc3ca4c1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bc3ca4c1

Branch: refs/heads/branch-2
Commit: bc3ca4c1061f222188007e3b07f64afc2b91ea35
Parents: 6a3929f
Author: Arpit Agarwal 
Authored: Thu Oct 5 15:26:52 2017 -0700
Committer: vrushali 
Committed: Fri Oct 20 11:22:27 2017 -0700

--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |  2 ++
 .../hadoop/hdfs/server/namenode/NameNode.java   | 16 +
 .../namenode/NameNodeFormatException.java   | 37 
 .../src/main/resources/hdfs-default.xml | 11 ++
 .../src/site/markdown/HDFSCommands.md   |  2 +-
 .../hdfs/server/namenode/TestClusterId.java | 34 ++
 6 files changed, 101 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc3ca4c1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 6bec228..e4c02c2 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -244,6 +244,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final String  DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT = 
"supergroup";
   public static final String  DFS_NAMENODE_ACLS_ENABLED_KEY = 
"dfs.namenode.acls.enabled";
   public static final boolean DFS_NAMENODE_ACLS_ENABLED_DEFAULT = false;
+  public static final String DFS_REFORMAT_DISABLED = "dfs.reformat.disabled";
+  public static final boolean DFS_REFORMAT_DISABLED_DEFAULT = false;
   public static final String  DFS_NAMENODE_XATTRS_ENABLED_KEY = 
"dfs.namenode.xattrs.enabled";
   public static final boolean DFS_NAMENODE_XATTRS_ENABLED_DEFAULT = true;
   public static final String  DFS_ADMIN = "dfs.cluster.administrators";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc3ca4c1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index f4097bb..1826bce 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -50,6 +50,7 @@ import 
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
 import 
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
 import org.apache.hadoop.hdfs.server.common.MetricsLoggerTask;
+import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState;
 import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby;
 import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
@@ -1149,6 +1150,21 @@ public class NameNode extends ReconfigurableBase 
implements
   FSNamesystem fsn = new FSNamesystem(conf, fsImage);
   fsImage.getEditLog().initJournalsForWrite();
 
+  // Abort NameNode format if reformat is disabled and if
+  // meta-dir already exists
+  if (conf.getBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED,
+  DFSConfigKeys.DFS_REFORMAT_DISABLED_DEFAULT)) {
+force = false;
+isInteractive = false;
+for (StorageDirectory sd : fsImage.storage.dirIterable(null)) {
+  if (sd.hasSomeData()) {
+throw new NameNodeFormatException(
+"NameNode format aborted as reformat is disabled for "
++ "this cluster.");
+  }
+}
+  }
+
   if (!fsImage.confirmFormat(force, isInteractive)) {
 return true; // aborted
   }

[09/50] hadoop git commit: HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by Ajay Kumar.

2017-10-20 Thread vrushali 
HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by 
Ajay Kumar.

(cherry picked from commit b6942cbe9b8c9469e8c2b64c3268d671f5a43e75)
(cherry picked from commit 5897095d539be086ed37df011f024e37eb37b0cd)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bc3ca4c1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bc3ca4c1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bc3ca4c1

Branch: refs/heads/YARN-3368_branch2
Commit: bc3ca4c1061f222188007e3b07f64afc2b91ea35
Parents: 6a3929f
Author: Arpit Agarwal 
Authored: Thu Oct 5 15:26:52 2017 -0700
Committer: vrushali 
Committed: Fri Oct 20 11:22:27 2017 -0700

--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |  2 ++
 .../hadoop/hdfs/server/namenode/NameNode.java   | 16 +
 .../namenode/NameNodeFormatException.java   | 37 
 .../src/main/resources/hdfs-default.xml | 11 ++
 .../src/site/markdown/HDFSCommands.md   |  2 +-
 .../hdfs/server/namenode/TestClusterId.java | 34 ++
 6 files changed, 101 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc3ca4c1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 6bec228..e4c02c2 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -244,6 +244,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final String  DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT = 
"supergroup";
   public static final String  DFS_NAMENODE_ACLS_ENABLED_KEY = 
"dfs.namenode.acls.enabled";
   public static final boolean DFS_NAMENODE_ACLS_ENABLED_DEFAULT = false;
+  public static final String DFS_REFORMAT_DISABLED = "dfs.reformat.disabled";
+  public static final boolean DFS_REFORMAT_DISABLED_DEFAULT = false;
   public static final String  DFS_NAMENODE_XATTRS_ENABLED_KEY = 
"dfs.namenode.xattrs.enabled";
   public static final boolean DFS_NAMENODE_XATTRS_ENABLED_DEFAULT = true;
   public static final String  DFS_ADMIN = "dfs.cluster.administrators";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc3ca4c1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index f4097bb..1826bce 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -50,6 +50,7 @@ import 
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
 import 
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
 import org.apache.hadoop.hdfs.server.common.MetricsLoggerTask;
+import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState;
 import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby;
 import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
@@ -1149,6 +1150,21 @@ public class NameNode extends ReconfigurableBase 
implements
   FSNamesystem fsn = new FSNamesystem(conf, fsImage);
   fsImage.getEditLog().initJournalsForWrite();
 
+  // Abort NameNode format if reformat is disabled and if
+  // meta-dir already exists
+  if (conf.getBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED,
+  DFSConfigKeys.DFS_REFORMAT_DISABLED_DEFAULT)) {
+force = false;
+isInteractive = false;
+for (StorageDirectory sd : fsImage.storage.dirIterable(null)) {
+  if (sd.hasSomeData()) {
+throw new NameNodeFormatException(
+"NameNode format aborted as reformat is disabled for "
++ "this cluster.");
+  }
+}
+  }
+
   if (!fsImage.confirmFormat(force, isInteractive)) {
 return true; // aborted
   }

[04/50] [abbrv] hadoop git commit: HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by Ajay Kumar.

2017-10-19 Thread templedf 
HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by 
Ajay Kumar.

(cherry picked from commit b6942cbe9b8c9469e8c2b64c3268d671f5a43e75)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5897095d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5897095d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5897095d

Branch: refs/heads/resource-types
Commit: 5897095d539be086ed37df011f024e37eb37b0cd
Parents: 175c447
Author: Arpit Agarwal 
Authored: Thu Oct 5 15:26:52 2017 -0700
Committer: Zhe Zhang 
Committed: Tue Oct 17 16:45:21 2017 -0700

--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |  3 ++
 .../hadoop/hdfs/server/namenode/NameNode.java   | 16 +
 .../namenode/NameNodeFormatException.java   | 37 
 .../src/main/resources/hdfs-default.xml | 10 ++
 .../src/site/markdown/HDFSCommands.md   |  2 +-
 .../hdfs/server/namenode/TestClusterId.java | 34 ++
 6 files changed, 101 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5897095d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index fbc49fa..582ae4e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -281,6 +281,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   "dfs.namenode.posix.acl.inheritance.enabled";
   public static final boolean
   DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_DEFAULT = true;
+  public static final String DFS_REFORMAT_DISABLED = "dfs.reformat.disabled";
+  public static final boolean DFS_REFORMAT_DISABLED_DEFAULT = false;
+
   public static final String  DFS_NAMENODE_XATTRS_ENABLED_KEY = 
"dfs.namenode.xattrs.enabled";
   public static final boolean DFS_NAMENODE_XATTRS_ENABLED_DEFAULT = true;
   public static final String  DFS_ADMIN = "dfs.cluster.administrators";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5897095d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 79bbbc5..32b873b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -50,6 +50,7 @@ import 
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
 import 
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
 import org.apache.hadoop.hdfs.server.common.MetricsLoggerTask;
+import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState;
 import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby;
 import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
@@ -1146,6 +1147,21 @@ public class NameNode extends ReconfigurableBase 
implements
   FSNamesystem fsn = new FSNamesystem(conf, fsImage);
   fsImage.getEditLog().initJournalsForWrite();
 
+  // Abort NameNode format if reformat is disabled and if
+  // meta-dir already exists
+  if (conf.getBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED,
+  DFSConfigKeys.DFS_REFORMAT_DISABLED_DEFAULT)) {
+force = false;
+isInteractive = false;
+for (StorageDirectory sd : fsImage.storage.dirIterable(null)) {
+  if (sd.hasSomeData()) {
+throw new NameNodeFormatException(
+"NameNode format aborted as reformat is disabled for "
++ "this cluster.");
+  }
+}
+  }
+
   if (!fsImage.confirmFormat(force, isInteractive)) {
 return true; // aborted
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5897095d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeFormatException.java
--
diff --git

hadoop git commit: HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by Ajay Kumar.

2017-10-18 Thread zhz 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 754a375d7 -> c06851235


HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by 
Ajay Kumar.

(cherry picked from commit b6942cbe9b8c9469e8c2b64c3268d671f5a43e75)
(cherry picked from commit 5897095d539be086ed37df011f024e37eb37b0cd)
(cherry picked from commit 0aa1b62b396f6fa43f15900c26dd28b95c4e40ce)
(cherry picked from commit dcdb3f250cd6b36f8a15070792c94dc854a6a42d)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0685123
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0685123
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0685123

Branch: refs/heads/branch-2.7
Commit: c06851235a910060e841d45e91529a232ff7c0ab
Parents: 754a375
Author: Arpit Agarwal 
Authored: Thu Oct 5 15:26:52 2017 -0700
Committer: Zhe Zhang 
Committed: Tue Oct 17 23:57:11 2017 -0700

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 ++
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |  2 ++
 .../hadoop/hdfs/server/namenode/NameNode.java   | 16 +
 .../namenode/NameNodeFormatException.java   | 37 
 .../src/main/resources/hdfs-default.xml | 10 ++
 .../src/site/markdown/HDFSCommands.md   |  2 +-
 .../hdfs/server/namenode/TestClusterId.java | 34 ++
 7 files changed, 103 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0685123/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 250e6dd..8eb5e7a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -14,6 +14,9 @@ Release 2.7.5 - UNRELEASED
 
 HDFS-12131. Add some of the FSNamesystem JMX values as metrics.
 (Erik Krogen via wang, shv)
+
+HDFS-12420. Add an option to disallow 'namenode format -force'.
+(Ajay Kumar via Arpit Agarwal, zhz)
 
   OPTIMIZATIONS
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0685123/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 98bc0ef..e4e72e6 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -249,6 +249,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final String  DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT = 
"supergroup";
   public static final String  DFS_NAMENODE_ACLS_ENABLED_KEY = 
"dfs.namenode.acls.enabled";
   public static final boolean DFS_NAMENODE_ACLS_ENABLED_DEFAULT = false;
+  public static final String DFS_REFORMAT_DISABLED = "dfs.reformat.disabled";
+  public static final boolean DFS_REFORMAT_DISABLED_DEFAULT = false;
   public static final String  DFS_NAMENODE_XATTRS_ENABLED_KEY = 
"dfs.namenode.xattrs.enabled";
   public static final boolean DFS_NAMENODE_XATTRS_ENABLED_DEFAULT = true;
   public static final String  DFS_ADMIN = "dfs.cluster.administrators";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0685123/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index b90be7c..897df65 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
 import 
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
+import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState;
 import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby;
 import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
@@ -992,6 +993,21 @@ public class NameNode

hadoop git commit: HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by Ajay Kumar.

2017-10-18 Thread zhz 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 a1d29717a -> dcdb3f250


HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by 
Ajay Kumar.

(cherry picked from commit b6942cbe9b8c9469e8c2b64c3268d671f5a43e75)
(cherry picked from commit 5897095d539be086ed37df011f024e37eb37b0cd)
(cherry picked from commit 0aa1b62b396f6fa43f15900c26dd28b95c4e40ce)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dcdb3f25
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dcdb3f25
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dcdb3f25

Branch: refs/heads/branch-2.8
Commit: dcdb3f250cd6b36f8a15070792c94dc854a6a42d
Parents: a1d2971
Author: Arpit Agarwal 
Authored: Thu Oct 5 15:26:52 2017 -0700
Committer: Zhe Zhang 
Committed: Tue Oct 17 23:46:14 2017 -0700

--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |  2 ++
 .../hadoop/hdfs/server/namenode/NameNode.java   | 16 +
 .../namenode/NameNodeFormatException.java   | 37 
 .../src/main/resources/hdfs-default.xml | 10 ++
 .../src/site/markdown/HDFSCommands.md   |  2 +-
 .../hdfs/server/namenode/TestClusterId.java | 34 ++
 6 files changed, 100 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcdb3f25/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index d262cb9..2139921 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -235,6 +235,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final String  DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT = 
"supergroup";
   public static final String  DFS_NAMENODE_ACLS_ENABLED_KEY = 
"dfs.namenode.acls.enabled";
   public static final boolean DFS_NAMENODE_ACLS_ENABLED_DEFAULT = false;
+  public static final String DFS_REFORMAT_DISABLED = "dfs.reformat.disabled";
+  public static final boolean DFS_REFORMAT_DISABLED_DEFAULT = false;
   public static final String  DFS_NAMENODE_XATTRS_ENABLED_KEY = 
"dfs.namenode.xattrs.enabled";
   public static final boolean DFS_NAMENODE_XATTRS_ENABLED_DEFAULT = true;
   public static final String  DFS_ADMIN = "dfs.cluster.administrators";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcdb3f25/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index a44df77..fd72eb6 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -45,6 +45,7 @@ import 
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
 import 
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
 import org.apache.hadoop.hdfs.server.common.MetricsLoggerTask;
+import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState;
 import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby;
 import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
@@ -1102,6 +1103,21 @@ public class NameNode implements NameNodeStatusMXBean {
   FSNamesystem fsn = new FSNamesystem(conf, fsImage);
   fsImage.getEditLog().initJournalsForWrite();
 
+  // Abort NameNode format if reformat is disabled and if
+  // meta-dir already exists
+  if (conf.getBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED,
+  DFSConfigKeys.DFS_REFORMAT_DISABLED_DEFAULT)) {
+force = false;
+isInteractive = false;
+for (StorageDirectory sd : fsImage.storage.dirIterable(null)) {
+  if (sd.hasSomeData()) {
+throw new NameNodeFormatException(
+"NameNode format aborted as reformat is disabled for "
++ "this cluster.");
+  }
+}
+  }
+
   if (!fsImage.confirmFormat(force, isInteractive)) {
 return true; //

hadoop git commit: HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by Ajay Kumar.

2017-10-18 Thread zhz 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 d894f910d -> 0aa1b62b3


HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by 
Ajay Kumar.

(cherry picked from commit b6942cbe9b8c9469e8c2b64c3268d671f5a43e75)
(cherry picked from commit 5897095d539be086ed37df011f024e37eb37b0cd)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0aa1b62b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0aa1b62b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0aa1b62b

Branch: refs/heads/branch-2
Commit: 0aa1b62b396f6fa43f15900c26dd28b95c4e40ce
Parents: d894f91
Author: Arpit Agarwal 
Authored: Thu Oct 5 15:26:52 2017 -0700
Committer: Zhe Zhang 
Committed: Tue Oct 17 23:44:13 2017 -0700

--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |  2 ++
 .../hadoop/hdfs/server/namenode/NameNode.java   | 16 +
 .../namenode/NameNodeFormatException.java   | 37 
 .../src/main/resources/hdfs-default.xml | 11 ++
 .../src/site/markdown/HDFSCommands.md   |  2 +-
 .../hdfs/server/namenode/TestClusterId.java | 34 ++
 6 files changed, 101 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aa1b62b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 6bec228..e4c02c2 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -244,6 +244,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final String  DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT = 
"supergroup";
   public static final String  DFS_NAMENODE_ACLS_ENABLED_KEY = 
"dfs.namenode.acls.enabled";
   public static final boolean DFS_NAMENODE_ACLS_ENABLED_DEFAULT = false;
+  public static final String DFS_REFORMAT_DISABLED = "dfs.reformat.disabled";
+  public static final boolean DFS_REFORMAT_DISABLED_DEFAULT = false;
   public static final String  DFS_NAMENODE_XATTRS_ENABLED_KEY = 
"dfs.namenode.xattrs.enabled";
   public static final boolean DFS_NAMENODE_XATTRS_ENABLED_DEFAULT = true;
   public static final String  DFS_ADMIN = "dfs.cluster.administrators";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aa1b62b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index f4097bb..1826bce 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -50,6 +50,7 @@ import 
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
 import 
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
 import org.apache.hadoop.hdfs.server.common.MetricsLoggerTask;
+import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState;
 import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby;
 import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
@@ -1149,6 +1150,21 @@ public class NameNode extends ReconfigurableBase 
implements
   FSNamesystem fsn = new FSNamesystem(conf, fsImage);
   fsImage.getEditLog().initJournalsForWrite();
 
+  // Abort NameNode format if reformat is disabled and if
+  // meta-dir already exists
+  if (conf.getBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED,
+  DFSConfigKeys.DFS_REFORMAT_DISABLED_DEFAULT)) {
+force = false;
+isInteractive = false;
+for (StorageDirectory sd : fsImage.storage.dirIterable(null)) {
+  if (sd.hasSomeData()) {
+throw new NameNodeFormatException(
+"NameNode format aborted as reformat is disabled for "
++ "this cluster.");
+  }
+}
+  }
+
   if (!fsImage.confirmFormat(force, isInteractive)) {
 return true; // aborted
   }

hadoop git commit: HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by Ajay Kumar.

2017-10-17 Thread zhz 
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 175c4472d -> 5897095d5


HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by 
Ajay Kumar.

(cherry picked from commit b6942cbe9b8c9469e8c2b64c3268d671f5a43e75)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5897095d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5897095d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5897095d

Branch: refs/heads/branch-3.0
Commit: 5897095d539be086ed37df011f024e37eb37b0cd
Parents: 175c447
Author: Arpit Agarwal 
Authored: Thu Oct 5 15:26:52 2017 -0700
Committer: Zhe Zhang 
Committed: Tue Oct 17 16:45:21 2017 -0700

--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |  3 ++
 .../hadoop/hdfs/server/namenode/NameNode.java   | 16 +
 .../namenode/NameNodeFormatException.java   | 37 
 .../src/main/resources/hdfs-default.xml | 10 ++
 .../src/site/markdown/HDFSCommands.md   |  2 +-
 .../hdfs/server/namenode/TestClusterId.java | 34 ++
 6 files changed, 101 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5897095d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index fbc49fa..582ae4e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -281,6 +281,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   "dfs.namenode.posix.acl.inheritance.enabled";
   public static final boolean
   DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_DEFAULT = true;
+  public static final String DFS_REFORMAT_DISABLED = "dfs.reformat.disabled";
+  public static final boolean DFS_REFORMAT_DISABLED_DEFAULT = false;
+
   public static final String  DFS_NAMENODE_XATTRS_ENABLED_KEY = 
"dfs.namenode.xattrs.enabled";
   public static final boolean DFS_NAMENODE_XATTRS_ENABLED_DEFAULT = true;
   public static final String  DFS_ADMIN = "dfs.cluster.administrators";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5897095d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 79bbbc5..32b873b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -50,6 +50,7 @@ import 
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
 import 
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
 import org.apache.hadoop.hdfs.server.common.MetricsLoggerTask;
+import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState;
 import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby;
 import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
@@ -1146,6 +1147,21 @@ public class NameNode extends ReconfigurableBase 
implements
   FSNamesystem fsn = new FSNamesystem(conf, fsImage);
   fsImage.getEditLog().initJournalsForWrite();
 
+  // Abort NameNode format if reformat is disabled and if
+  // meta-dir already exists
+  if (conf.getBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED,
+  DFSConfigKeys.DFS_REFORMAT_DISABLED_DEFAULT)) {
+force = false;
+isInteractive = false;
+for (StorageDirectory sd : fsImage.storage.dirIterable(null)) {
+  if (sd.hasSomeData()) {
+throw new NameNodeFormatException(
+"NameNode format aborted as reformat is disabled for "
++ "this cluster.");
+  }
+}
+  }
+
   if (!fsImage.confirmFormat(force, isInteractive)) {
 return true; // aborted
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5897095d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeFormatException.java

[06/45] hadoop git commit: HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by Ajay Kumar.

2017-10-06 Thread inigoiri 
HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by 
Ajay Kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6942cbe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6942cbe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6942cbe

Branch: refs/heads/HDFS-10467
Commit: b6942cbe9b8c9469e8c2b64c3268d671f5a43e75
Parents: d0046bb
Author: Arpit Agarwal 
Authored: Thu Oct 5 15:26:52 2017 -0700
Committer: Arpit Agarwal 
Committed: Thu Oct 5 15:26:52 2017 -0700

--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |  3 ++
 .../hadoop/hdfs/server/namenode/NameNode.java   | 16 +
 .../namenode/NameNodeFormatException.java   | 37 
 .../src/main/resources/hdfs-default.xml | 10 ++
 .../src/site/markdown/HDFSCommands.md   |  2 +-
 .../hdfs/server/namenode/TestClusterId.java | 34 ++
 6 files changed, 101 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index b4842f9..f2f8730 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -270,6 +270,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   "dfs.namenode.posix.acl.inheritance.enabled";
   public static final boolean
   DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_DEFAULT = true;
+  public static final String DFS_REFORMAT_DISABLED = "dfs.reformat.disabled";
+  public static final boolean DFS_REFORMAT_DISABLED_DEFAULT = false;
+
   public static final String  DFS_NAMENODE_XATTRS_ENABLED_KEY = 
"dfs.namenode.xattrs.enabled";
   public static final boolean DFS_NAMENODE_XATTRS_ENABLED_DEFAULT = true;
   public static final String  DFS_ADMIN = "dfs.cluster.administrators";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 79bbbc5..32b873b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -50,6 +50,7 @@ import 
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
 import 
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
 import org.apache.hadoop.hdfs.server.common.MetricsLoggerTask;
+import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState;
 import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby;
 import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
@@ -1146,6 +1147,21 @@ public class NameNode extends ReconfigurableBase 
implements
   FSNamesystem fsn = new FSNamesystem(conf, fsImage);
   fsImage.getEditLog().initJournalsForWrite();
 
+  // Abort NameNode format if reformat is disabled and if
+  // meta-dir already exists
+  if (conf.getBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED,
+  DFSConfigKeys.DFS_REFORMAT_DISABLED_DEFAULT)) {
+force = false;
+isInteractive = false;
+for (StorageDirectory sd : fsImage.storage.dirIterable(null)) {
+  if (sd.hasSomeData()) {
+throw new NameNodeFormatException(
+"NameNode format aborted as reformat is disabled for "
++ "this cluster.");
+  }
+}
+  }
+
   if (!fsImage.confirmFormat(force, isInteractive)) {
 return true; // aborted
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeFormatException.java
--
diff --git

[25/50] [abbrv] hadoop git commit: HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by Ajay Kumar.

2017-10-06 Thread jhung 
HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by 
Ajay Kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6942cbe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6942cbe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6942cbe

Branch: refs/heads/YARN-5734
Commit: b6942cbe9b8c9469e8c2b64c3268d671f5a43e75
Parents: d0046bb
Author: Arpit Agarwal 
Authored: Thu Oct 5 15:26:52 2017 -0700
Committer: Arpit Agarwal 
Committed: Thu Oct 5 15:26:52 2017 -0700

--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |  3 ++
 .../hadoop/hdfs/server/namenode/NameNode.java   | 16 +
 .../namenode/NameNodeFormatException.java   | 37 
 .../src/main/resources/hdfs-default.xml | 10 ++
 .../src/site/markdown/HDFSCommands.md   |  2 +-
 .../hdfs/server/namenode/TestClusterId.java | 34 ++
 6 files changed, 101 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index b4842f9..f2f8730 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -270,6 +270,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   "dfs.namenode.posix.acl.inheritance.enabled";
   public static final boolean
   DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_DEFAULT = true;
+  public static final String DFS_REFORMAT_DISABLED = "dfs.reformat.disabled";
+  public static final boolean DFS_REFORMAT_DISABLED_DEFAULT = false;
+
   public static final String  DFS_NAMENODE_XATTRS_ENABLED_KEY = 
"dfs.namenode.xattrs.enabled";
   public static final boolean DFS_NAMENODE_XATTRS_ENABLED_DEFAULT = true;
   public static final String  DFS_ADMIN = "dfs.cluster.administrators";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 79bbbc5..32b873b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -50,6 +50,7 @@ import 
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
 import 
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
 import org.apache.hadoop.hdfs.server.common.MetricsLoggerTask;
+import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState;
 import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby;
 import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
@@ -1146,6 +1147,21 @@ public class NameNode extends ReconfigurableBase 
implements
   FSNamesystem fsn = new FSNamesystem(conf, fsImage);
   fsImage.getEditLog().initJournalsForWrite();
 
+  // Abort NameNode format if reformat is disabled and if
+  // meta-dir already exists
+  if (conf.getBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED,
+  DFSConfigKeys.DFS_REFORMAT_DISABLED_DEFAULT)) {
+force = false;
+isInteractive = false;
+for (StorageDirectory sd : fsImage.storage.dirIterable(null)) {
+  if (sd.hasSomeData()) {
+throw new NameNodeFormatException(
+"NameNode format aborted as reformat is disabled for "
++ "this cluster.");
+  }
+}
+  }
+
   if (!fsImage.confirmFormat(force, isInteractive)) {
 return true; // aborted
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeFormatException.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeFormatException.java

hadoop git commit: HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by Ajay Kumar.

2017-10-05 Thread arp 
Repository: hadoop
Updated Branches:
  refs/heads/trunk d0046bb5c -> b6942cbe9


HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by 
Ajay Kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6942cbe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6942cbe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6942cbe

Branch: refs/heads/trunk
Commit: b6942cbe9b8c9469e8c2b64c3268d671f5a43e75
Parents: d0046bb
Author: Arpit Agarwal 
Authored: Thu Oct 5 15:26:52 2017 -0700
Committer: Arpit Agarwal 
Committed: Thu Oct 5 15:26:52 2017 -0700

--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |  3 ++
 .../hadoop/hdfs/server/namenode/NameNode.java   | 16 +
 .../namenode/NameNodeFormatException.java   | 37 
 .../src/main/resources/hdfs-default.xml | 10 ++
 .../src/site/markdown/HDFSCommands.md   |  2 +-
 .../hdfs/server/namenode/TestClusterId.java | 34 ++
 6 files changed, 101 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index b4842f9..f2f8730 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -270,6 +270,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   "dfs.namenode.posix.acl.inheritance.enabled";
   public static final boolean
   DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_DEFAULT = true;
+  public static final String DFS_REFORMAT_DISABLED = "dfs.reformat.disabled";
+  public static final boolean DFS_REFORMAT_DISABLED_DEFAULT = false;
+
   public static final String  DFS_NAMENODE_XATTRS_ENABLED_KEY = 
"dfs.namenode.xattrs.enabled";
   public static final boolean DFS_NAMENODE_XATTRS_ENABLED_DEFAULT = true;
   public static final String  DFS_ADMIN = "dfs.cluster.administrators";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 79bbbc5..32b873b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -50,6 +50,7 @@ import 
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
 import 
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
 import org.apache.hadoop.hdfs.server.common.MetricsLoggerTask;
+import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState;
 import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby;
 import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
@@ -1146,6 +1147,21 @@ public class NameNode extends ReconfigurableBase 
implements
   FSNamesystem fsn = new FSNamesystem(conf, fsImage);
   fsImage.getEditLog().initJournalsForWrite();
 
+  // Abort NameNode format if reformat is disabled and if
+  // meta-dir already exists
+  if (conf.getBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED,
+  DFSConfigKeys.DFS_REFORMAT_DISABLED_DEFAULT)) {
+force = false;
+isInteractive = false;
+for (StorageDirectory sd : fsImage.storage.dirIterable(null)) {
+  if (sd.hasSomeData()) {
+throw new NameNodeFormatException(
+"NameNode format aborted as reformat is disabled for "
++ "this cluster.");
+  }
+}
+  }
+
   if (!fsImage.confirmFormat(force, isInteractive)) {
 return true; // aborted
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeFormatException.java
--
diff --git