Author: todd Date: Thu Apr 26 00:15:47 2012 New Revision: 1330611 URL: http://svn.apache.org/viewvc?rev=1330611&view=rev Log: HDFS-3094. add -nonInteractive and -force option to namenode -format command. Contributed by Arpit Gupta.
Added: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestNameNodeFormat.java Modified: hadoop/common/branches/branch-1/CHANGES.txt hadoop/common/branches/branch-1/src/docs/src/documentation/content/xdocs/commands_manual.xml hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/common/HdfsConstants.java hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java Modified: hadoop/common/branches/branch-1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1330611&r1=1330610&r2=1330611&view=diff ============================================================================== --- hadoop/common/branches/branch-1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1/CHANGES.txt Thu Apr 26 00:15:47 2012 @@ -96,6 +96,9 @@ Release 1.1.0 - unreleased HDFS-1378. Edit log replay should track and report file offsets in case of errors. (atm and todd, backport by Colin Patrick McCabe via todd) + HDFS-3094. add -nonInteractive and -force option to namenode -format + command. (Arpit Gupta via todd) + BUG FIXES MAPREDUCE-4087. [Gridmix] GenerateDistCacheData job of Gridmix can Modified: hadoop/common/branches/branch-1/src/docs/src/documentation/content/xdocs/commands_manual.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/docs/src/documentation/content/xdocs/commands_manual.xml?rev=1330611&r1=1330610&r2=1330611&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/docs/src/documentation/content/xdocs/commands_manual.xml (original) +++ hadoop/common/branches/branch-1/src/docs/src/documentation/content/xdocs/commands_manual.xml Thu Apr 26 00:15:47 2012 @@ -631,14 +631,16 @@ <a href="hdfs_user_guide.html#Upgrade+and+Rollback">Upgrade Rollback</a> </p> <p> - <code>Usage: hadoop namenode [-format] | [-upgrade] | [-rollback] | [-finalize] | [-importCheckpoint]</code> + <code>Usage: hadoop namenode [-format [-force] [-nonInteractive]] | [-upgrade] | [-rollback] | [-finalize] | [-importCheckpoint]</code> </p> <table> <tr><th> COMMAND_OPTION </th><th> Description </th></tr> <tr> - <td><code>-format</code></td> - <td>Formats the namenode. It starts the namenode, formats it and then shut it down.</td> + <td><code>-format [-force] [-nonInteractive]</code></td> + <td>Formats the namenode. It starts the namenode, formats it and then shuts it down. User will be prompted for input if the name directories exist on the local filesystem.<br/> + -nonInteractive: User will not be prompted for input if the name directories exist in the local filesystem and the format will fail.<br/> + -force: formats the namenode and the user will NOT be prompted to confirm formatting of name directories in the local filesystem. If -nonInteractive option is specified it will be ignored.</td> </tr> <tr> <td><code>-upgrade</code></td> Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/common/HdfsConstants.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/common/HdfsConstants.java?rev=1330611&r1=1330610&r2=1330611&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/common/HdfsConstants.java (original) +++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/common/HdfsConstants.java Thu Apr 26 00:15:47 2012 @@ -43,11 +43,16 @@ public interface HdfsConstants { FORCE ("-force"), ROLLBACK("-rollback"), FINALIZE("-finalize"), - IMPORT ("-importCheckpoint"); + IMPORT ("-importCheckpoint"), + NONINTERACTIVE ("-nonInteractive"); // Used only with recovery option private int force = MetaRecoveryContext.FORCE_NONE; - + + // used only with format option + private boolean isConfirmationNeeded = true; + private boolean isInteractive = true; + private String name = null; private StartupOption(String arg) {this.name = arg;} public String getName() {return name;} @@ -65,6 +70,22 @@ public interface HdfsConstants { public int getForce() { return this.force; } + + public void setConfirmationNeeded(boolean confirmationNeeded) { + this.isConfirmationNeeded = confirmationNeeded; + } + + public boolean getConfirmationNeeded() { + return isConfirmationNeeded; + } + + public void setInteractive(boolean interactive) { + this.isInteractive = interactive; + } + + public boolean getInteractive() { + return isInteractive; + } } // Timeouts for communicating with DataNode for streaming writes/reads Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1330611&r1=1330610&r2=1330611&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original) +++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java Thu Apr 26 00:15:47 2012 @@ -181,7 +181,7 @@ public class NameNode implements ClientP /** Format a new filesystem. Destroys any filesystem that may already * exist at this location. **/ public static void format(Configuration conf) throws IOException { - format(conf, false); + format(conf, false, true); } static NameNodeInstrumentation myMetrics; @@ -1142,8 +1142,7 @@ public class NameNode implements ClientP * @throws IOException */ private static boolean format(Configuration conf, - boolean isConfirmationNeeded - ) throws IOException { + boolean isConfirmationNeeded, boolean isInteractive) throws IOException { Collection<File> dirsToFormat = FSNamesystem.getNamespaceDirs(conf); Collection<File> editDirsToFormat = FSNamesystem.getNamespaceEditsDirs(conf); @@ -1152,6 +1151,10 @@ public class NameNode implements ClientP if (!curDir.exists()) continue; if (isConfirmationNeeded) { + if (!isInteractive) { + System.err.println("Format aborted: " + curDir + " exists."); + return true; + } System.err.print("Re-format filesystem in " + curDir +" ? (Y or N) "); if (!(System.in.read() == 'Y')) { System.err.println("Format aborted in "+ curDir); @@ -1218,7 +1221,8 @@ public class NameNode implements ClientP private static void printUsage() { System.err.println( "Usage: java NameNode [" + - StartupOption.FORMAT.getName() + "] | [" + + StartupOption.FORMAT.getName() + " [" + StartupOption.FORCE.getName() + + " ] ["+StartupOption.NONINTERACTIVE.getName()+"]] | [" + StartupOption.UPGRADE.getName() + "] | [" + StartupOption.ROLLBACK.getName() + "] | [" + StartupOption.FINALIZE.getName() + "] | [" + @@ -1234,6 +1238,15 @@ public class NameNode implements ClientP String cmd = args[i]; if (StartupOption.FORMAT.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.FORMAT; + // check if there are other options + for (i = i + 1; i < argsLen; i++) { + if (args[i].equalsIgnoreCase(StartupOption.FORCE.getName())) { + startOpt.setConfirmationNeeded(false); + } + if (args[i].equalsIgnoreCase(StartupOption.NONINTERACTIVE.getName())) { + startOpt.setInteractive(false); + } + } } else if (StartupOption.REGULAR.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.REGULAR; } else if (StartupOption.UPGRADE.getName().equalsIgnoreCase(cmd)) { @@ -1344,7 +1357,8 @@ public class NameNode implements ClientP switch (startOpt) { case FORMAT: - boolean aborted = format(conf, true); + boolean aborted = format(conf, startOpt.getConfirmationNeeded(), + startOpt.getInteractive()); System.exit(aborted ? 1 : 0); case FINALIZE: aborted = finalize(conf, true); Added: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestNameNodeFormat.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestNameNodeFormat.java?rev=1330611&view=auto ============================================================================== --- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestNameNodeFormat.java (added) +++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestNameNodeFormat.java Thu Apr 26 00:15:47 2012 @@ -0,0 +1,298 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.security.Permission; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import junit.framework.TestCase; + +public class TestNameNodeFormat extends TestCase { + private static final Log LOG = LogFactory.getLog(TestNameNodeFormat.class); + File hdfsDir; + String baseDir; + Configuration config; + + @Before + public void setUp() throws IOException { + System.setSecurityManager(new NoExitSecurityManager()); + + baseDir = System.getProperty("test.build.data", "build/test/data"); + + hdfsDir = new File(baseDir, "dfs/name"); + if (hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir)) { + throw new IOException("Could not delete test directory '" + hdfsDir + "'"); + } + LOG.info("hdfsdir is " + hdfsDir.getAbsolutePath()); + + // set the defaults before every test to make sure we have a clean state + StartupOption.FORMAT.setConfirmationNeeded(true); + StartupOption.FORMAT.setInteractive(true); + + config = new Configuration(); + config.set("dfs.name.dir", hdfsDir.getPath()); + } + + @After + public void tearDown() throws IOException { + System.setSecurityManager(null); // or save and restore original + if (hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir)) { + throw new IOException("Could not tearDown test directory '" + hdfsDir + + "'"); + } + } + + /** + * Test namenode format with -format option. Format should succeed. + * + * @throws IOException + */ + @Test + public void testFormat() throws IOException { + + String[] argv = { "-format" }; + try { + NameNode.createNameNode(argv, config); + fail("createNameNode() did not call System.exit()"); + } catch (ExitException e) { + // catch the exit code and check the status + assertEquals("Format should have succeeded", 0, e.status); + } + // check if the version file exists. + File version = new File(hdfsDir, "current/VERSION"); + assertTrue("Check version file exists", version.exists()); + } + + /** + * Test namenode format with -format -force option when the name directory + * exists. Format should succeed. + * + * @throws IOException + */ + @Test + public void testFormatWithForce() throws IOException { + + // create the directory + if (!hdfsDir.mkdirs()) { + fail("Failed to create dir " + hdfsDir.getPath()); + } + + String[] argv = { "-format", "-force" }; + try { + NameNode.createNameNode(argv, config); + fail("createNameNode() did not call System.exit()"); + } catch (ExitException e) { + // catch the exit code and check the status + assertEquals("Format should have succeeded", 0, e.status); + } + + // check if the version file exists. + File version = new File(hdfsDir, "current/VERSION"); + assertTrue("Check version file exists", version.exists()); + } + + /** + * Test namenode format with -format -nonInteractive when the name directory + * exists. Format should be aborted. + * + * @throws IOException + */ + @Test + public void testFormatWithNonInteractive() throws IOException { + + // create the directory + if (!hdfsDir.mkdirs()) { + fail("Failed to create dir " + hdfsDir.getPath()); + } + + String[] argv = { "-format", "-nonInteractive" }; + try { + NameNode.createNameNode(argv, config); + fail("createNameNode() did not call System.exit()"); + } catch (ExitException e) { + // catch the exit code and check the status + assertEquals("Format should have been aborted with exit code 1", 1, + e.status); + } + + // check if the version file does not exists. + File version = new File(hdfsDir, "current/VERSION"); + assertFalse("Check version should not exist", version.exists()); + } + + /** + * Test namenode format with -format -nonInteractive when name directory does + * not exist. Format should succeed. + * + * @throws IOException + */ + @Test + public void testFormatWithNonInteractiveNameDirDoesNotExit() + throws IOException { + + String[] argv = { "-format", "-nonInteractive" }; + try { + NameNode.createNameNode(argv, config); + fail("createNameNode() did not call System.exit()"); + } catch (ExitException e) { + // catch the exit code and check the status + assertEquals("Format should have succeeded", 0, e.status); + } + // check if the version file exists. + File version = new File(hdfsDir, "current/VERSION"); + assertTrue("Check version file exists", version.exists()); + } + + /** + * Test namenode format with -format -nonInteractive -force when the name + * directory exists. Format should succeed. + * + * @throws IOException + */ + @Test + public void testFormatWithNonInteractiveAndForce() throws IOException { + + // create the directory + if (!hdfsDir.mkdirs()) { + fail("Failed to create dir " + hdfsDir.getPath()); + } + + String[] argv = { "-format", "-nonInteractive", "-force" }; + try { + NameNode.createNameNode(argv, config); + fail("createNameNode() did not call System.exit()"); + } catch (ExitException e) { + // catch the exit code and check the status + assertEquals("Format should have succeeded", 0, e.status); + } + + // check if the version file exists. + File version = new File(hdfsDir, "current/VERSION"); + assertTrue("Check version file exists", version.exists()); + } + + /** + * Test namenode format with -format when the name directory exists and enter + * N when prompted. Format should be aborted. + * + * @throws IOException + * @throws InterruptedException + */ + @Test + public void testFormatWithoutForceEnterN() throws IOException, + InterruptedException { + + // create the directory + if (!hdfsDir.mkdirs()) { + fail("Failed to create dir " + hdfsDir.getPath()); + } + + // capture the input stream + InputStream origIn = System.in; + ByteArrayInputStream bins = new ByteArrayInputStream("N\n".getBytes()); + System.setIn(bins); + String[] argv = { "-format" }; + try { + NameNode.createNameNode(argv, config); + fail("createNameNode() did not call System.exit()"); + } catch (ExitException e) { + assertEquals("Format should not have succeeded", 1, e.status); + } + + System.setIn(origIn); + + // check if the version file does not exists. + File version = new File(hdfsDir, "current/VERSION"); + assertFalse("Check version should not exist", version.exists()); + } + + /** + * Test namenode format with -format option when the name directory exists and + * enter Y when prompted. Format should succeed. + * + * @throws IOException + * @throws InterruptedException + */ + @Test + public void testFormatWithoutForceEnterY() throws IOException, + InterruptedException { + + // create the directory + if (!hdfsDir.mkdirs()) { + fail("Failed to create dir " + hdfsDir.getPath()); + } + + // capture the input stream + InputStream origIn = System.in; + ByteArrayInputStream bins = new ByteArrayInputStream("Y\n".getBytes()); + System.setIn(bins); + String[] argv = { "-format" }; + try { + NameNode.createNameNode(argv, config); + fail("createNameNode() did not call System.exit()"); + } catch (ExitException e) { + assertEquals("Format should have succeeded", 0, e.status); + } + + System.setIn(origIn); + + // check if the version file does exist. + File version = new File(hdfsDir, "current/VERSION"); + assertTrue("Check version file should exist", version.exists()); + } + + private static class ExitException extends SecurityException { + private static final long serialVersionUID = 1L; + public final int status; + + public ExitException(int status) { + super("There is no escape!"); + this.status = status; + } + } + + private static class NoExitSecurityManager extends SecurityManager { + @Override + public void checkPermission(Permission perm) { + // allow anything. + } + + @Override + public void checkPermission(Permission perm, Object context) { + // allow anything. + } + + @Override + public void checkExit(int status) { + super.checkExit(status); + throw new ExitException(status); + } + } +} \ No newline at end of file