hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject hadoop git commit: HADOOP-12043. Display warning if defaultFs is not set when running fs commands. Contributed by Eddy Xu.
Date Sat, 30 May 2015 00:16:27 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 6aec13cb3 -> 374ddd9f9


HADOOP-12043. Display warning if defaultFs is not set when running fs commands. Contributed
by Eddy Xu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/374ddd9f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/374ddd9f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/374ddd9f

Branch: refs/heads/trunk
Commit: 374ddd9f9ea43b0e730a7baab3e51e6893d40420
Parents: 6aec13c
Author: Andrew Wang <wang@apache.org>
Authored: Fri May 29 17:15:58 2015 -0700
Committer: Andrew Wang <wang@apache.org>
Committed: Fri May 29 17:15:58 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 ++
 .../fs/CommonConfigurationKeysPublic.java       |  6 ++++
 .../org/apache/hadoop/fs/shell/FsCommand.java   | 30 +++++++++++++++++++-
 .../java/org/apache/hadoop/fs/shell/Ls.java     |  8 ++++++
 .../src/main/resources/core-default.xml         |  8 ++++++
 .../java/org/apache/hadoop/fs/shell/TestLs.java | 25 ++++++++++++++++
 6 files changed, 79 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/374ddd9f/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index e5d2e39..00075b4 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -626,6 +626,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-11894. Bump the version of Apache HTrace to 3.2.0-incubating
     (Masatake Iwasaki via Colin P. McCabe)
 
+    HADOOP-12043. Display warning if defaultFs is not set when running fs
+    commands. (Lei Xu via wang)
+
   OPTIMIZATIONS
 
     HADOOP-11785. Reduce the number of listStatus operation in distcp

http://git-wip-us.apache.org/repos/asf/hadoop/blob/374ddd9f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index 90c6934..7231d59 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -363,5 +363,11 @@ public class CommonConfigurationKeysPublic {
     "hadoop.security.random.device.file.path";
   public static final String HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_DEFAULT = 
     "/dev/urandom";
+
+  /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+  public static final String HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY =
+      "hadoop.shell.missing.defaultFs.warning";
+  public static final boolean HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT =
+      false;
 }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/374ddd9f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
index 9515fde..971950f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.fs.shell;
 
 import java.io.IOException;
+import java.util.LinkedList;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -27,8 +28,13 @@ import org.apache.hadoop.fs.FsShellPermissions;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.shell.find.Find;
 
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY;
+
 /**
- * Base class for all "hadoop fs" commands
+ * Base class for all "hadoop fs" commands.
  */
 
 @InterfaceAudience.Private
@@ -90,4 +96,26 @@ abstract public class FsCommand extends Command {
   public int runAll() {
     return run(args);
   }
+
+  @Override
+  protected void processRawArguments(LinkedList<String> args)
+      throws IOException {
+    LinkedList<PathData> expendedArgs = expandArguments(args);
+    // If "fs.defaultFs" is not set appropriately, it warns the user that the
+    // command is not running against HDFS.
+    final boolean displayWarnings = getConf().getBoolean(
+        HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY,
+        HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT);
+    if (displayWarnings) {
+      final String defaultFs = getConf().get(FS_DEFAULT_NAME_KEY);
+      final boolean missingDefaultFs =
+          defaultFs == null || defaultFs.equals(FS_DEFAULT_NAME_DEFAULT);
+      if (missingDefaultFs) {
+        err.printf(
+            "Warning: fs.defaultFs is not set when running \"%s\" command.%n",
+            getCommandName());
+      }
+    }
+    processArguments(expendedArgs);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/374ddd9f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
index d5c52ed..67348c6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
@@ -24,6 +24,8 @@ import java.util.Arrays;
 import java.util.Comparator;
 import java.util.Date;
 import java.util.LinkedList;
+
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.StringUtils;
 
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -102,6 +104,12 @@ class Ls extends FsCommand {
 
   protected boolean humanReadable = false;
 
+  protected Ls() {}
+
+  protected Ls(Configuration conf) {
+    super(conf);
+  }
+
   protected String formatSize(long size) {
     return humanReadable
       ? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/374ddd9f/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index a1bc780..ae23a75 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1899,4 +1899,12 @@ for ldap providers in the same way as above does.
     <value>Client</value>
   </property>
 
+  <property>
+    <description>
+      Enable hdfs shell commands to display warnings if (fs.defaultFS) property
+      is not set.
+    </description>
+    <name>hadoop.shell.missing.defaultFs.warning</name>
+    <value>false</value>
+  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/374ddd9f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
index 4a9103f..cba44ee 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
@@ -17,10 +17,12 @@
  */
 package org.apache.hadoop.fs.shell;
 
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY;
 import static org.junit.Assert.*;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.*;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
 import java.net.URI;
@@ -1048,6 +1050,29 @@ public class TestLs {
     verifyNoMoreInteractions(out);
   }
 
+  private static void displayWarningOnLocalFileSystem(boolean shouldDisplay)
+      throws IOException {
+    Configuration conf = new Configuration();
+    conf.setBoolean(
+        HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY, shouldDisplay);
+
+    ByteArrayOutputStream buf = new ByteArrayOutputStream();
+    PrintStream err = new PrintStream(buf, true);
+    Ls ls = new Ls(conf);
+    ls.err = err;
+    ls.run("file:///.");
+    assertEquals(shouldDisplay, buf.toString().contains(
+        "Warning: fs.defaultFs is not set when running \"ls\" command."));
+  }
+
+  @Test
+  public void displayWarningsOnLocalFileSystem() throws IOException {
+    // Display warnings.
+    displayWarningOnLocalFileSystem(true);
+    // Does not display warnings.
+    displayWarningOnLocalFileSystem(false);
+  }
+
   // check the deprecated flag isn't set
   @Test
   public void isDeprecated() {


Mime
View raw message