HADOOP-12043. Display warning if defaultFs is not set when running fs commands. Contributed by Eddy Xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/312030a3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/312030a3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/312030a3 Branch: refs/heads/YARN-2928 Commit: 312030a3122d10b829c16ab655225128e5434ecb Parents: 704eee9 Author: Andrew Wang <w...@apache.org> Authored: Fri May 29 17:15:58 2015 -0700 Committer: Zhijie Shen <zjs...@apache.org> Committed: Tue Jun 2 16:12:58 2015 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++ .../fs/CommonConfigurationKeysPublic.java | 6 ++++ .../org/apache/hadoop/fs/shell/FsCommand.java | 30 +++++++++++++++++++- .../java/org/apache/hadoop/fs/shell/Ls.java | 8 ++++++ .../src/main/resources/core-default.xml | 8 ++++++ .../java/org/apache/hadoop/fs/shell/TestLs.java | 25 ++++++++++++++++ 6 files changed, 79 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/312030a3/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index e5d2e39..00075b4 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -626,6 +626,9 @@ Release 2.8.0 - UNRELEASED HADOOP-11894. Bump the version of Apache HTrace to 3.2.0-incubating (Masatake Iwasaki via Colin P. McCabe) + HADOOP-12043. Display warning if defaultFs is not set when running fs + commands. (Lei Xu via wang) + OPTIMIZATIONS HADOOP-11785. Reduce the number of listStatus operation in distcp http://git-wip-us.apache.org/repos/asf/hadoop/blob/312030a3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 90c6934..7231d59 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -363,5 +363,11 @@ public class CommonConfigurationKeysPublic { "hadoop.security.random.device.file.path"; public static final String HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_DEFAULT = "/dev/urandom"; + + /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */ + public static final String HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY = + "hadoop.shell.missing.defaultFs.warning"; + public static final boolean HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT = + false; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/312030a3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java index 9515fde..971950f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.shell; import java.io.IOException; +import java.util.LinkedList; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -27,8 +28,13 @@ import org.apache.hadoop.fs.FsShellPermissions; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.shell.find.Find; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY; + /** - * Base class for all "hadoop fs" commands + * Base class for all "hadoop fs" commands. */ @InterfaceAudience.Private @@ -90,4 +96,26 @@ abstract public class FsCommand extends Command { public int runAll() { return run(args); } + + @Override + protected void processRawArguments(LinkedList<String> args) + throws IOException { + LinkedList<PathData> expendedArgs = expandArguments(args); + // If "fs.defaultFs" is not set appropriately, it warns the user that the + // command is not running against HDFS. + final boolean displayWarnings = getConf().getBoolean( + HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY, + HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT); + if (displayWarnings) { + final String defaultFs = getConf().get(FS_DEFAULT_NAME_KEY); + final boolean missingDefaultFs = + defaultFs == null || defaultFs.equals(FS_DEFAULT_NAME_DEFAULT); + if (missingDefaultFs) { + err.printf( + "Warning: fs.defaultFs is not set when running \"%s\" command.%n", + getCommandName()); + } + } + processArguments(expendedArgs); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/312030a3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java index d5c52ed..67348c6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java @@ -24,6 +24,8 @@ import java.util.Arrays; import java.util.Comparator; import java.util.Date; import java.util.LinkedList; + +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; @@ -102,6 +104,12 @@ class Ls extends FsCommand { protected boolean humanReadable = false; + protected Ls() {} + + protected Ls(Configuration conf) { + super(conf); + } + protected String formatSize(long size) { return humanReadable ? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1) http://git-wip-us.apache.org/repos/asf/hadoop/blob/312030a3/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index a1bc780..ae23a75 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1899,4 +1899,12 @@ for ldap providers in the same way as above does. <value>Client</value> </property> + <property> + <description> + Enable hdfs shell commands to display warnings if (fs.defaultFS) property + is not set. + </description> + <name>hadoop.shell.missing.defaultFs.warning</name> + <value>false</value> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/312030a3/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java index 4a9103f..cba44ee 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java @@ -17,10 +17,12 @@ */ package org.apache.hadoop.fs.shell; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY; import static org.junit.Assert.*; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.*; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; import java.net.URI; @@ -1048,6 +1050,29 @@ public class TestLs { verifyNoMoreInteractions(out); } + private static void displayWarningOnLocalFileSystem(boolean shouldDisplay) + throws IOException { + Configuration conf = new Configuration(); + conf.setBoolean( + HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY, shouldDisplay); + + ByteArrayOutputStream buf = new ByteArrayOutputStream(); + PrintStream err = new PrintStream(buf, true); + Ls ls = new Ls(conf); + ls.err = err; + ls.run("file:///."); + assertEquals(shouldDisplay, buf.toString().contains( + "Warning: fs.defaultFs is not set when running \"ls\" command.")); + } + + @Test + public void displayWarningsOnLocalFileSystem() throws IOException { + // Display warnings. + displayWarningOnLocalFileSystem(true); + // Does not display warnings. + displayWarningOnLocalFileSystem(false); + } + // check the deprecated flag isn't set @Test public void isDeprecated() {