hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject svn commit: r1126769 - in /hadoop/common/branches/yahoo-merge: ./ src/contrib/ec2/ src/docs/ src/java/ src/java/org/apache/hadoop/fs/ src/java/org/apache/hadoop/fs/shell/ src/test/core/ src/test/core/org/apache/hadoop/fs/shell/
Date Mon, 23 May 2011 21:46:50 GMT
Author: szetszwo
Date: Mon May 23 21:46:49 2011
New Revision: 1126769

URL: http://svn.apache.org/viewvc?rev=1126769&view=rev
Log:
Merge 1091902 and 1091970 from trunk for HADOOP-7224.

Added:
    hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/CommandFactory.java
      - copied unchanged from r1091902, hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/CommandFactory.java
    hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/fs/shell/TestCommandFactory.java
      - copied unchanged from r1091970, hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/shell/TestCommandFactory.java
Modified:
    hadoop/common/branches/yahoo-merge/   (props changed)
    hadoop/common/branches/yahoo-merge/CHANGES.txt   (contents, props changed)
    hadoop/common/branches/yahoo-merge/src/contrib/ec2/   (props changed)
    hadoop/common/branches/yahoo-merge/src/docs/   (props changed)
    hadoop/common/branches/yahoo-merge/src/java/   (props changed)
    hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java
    hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/Command.java
    hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/Count.java
    hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java
    hadoop/common/branches/yahoo-merge/src/test/core/   (props changed)

Propchange: hadoop/common/branches/yahoo-merge/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 21:46:49 2011
@@ -1,2 +1,2 @@
-/hadoop/common/trunk:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1092565,1100026
+/hadoop/common/trunk:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092565,1100026
 /hadoop/core/branches/branch-0.19/core:713112

Modified: hadoop/common/branches/yahoo-merge/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/CHANGES.txt?rev=1126769&r1=1126768&r2=1126769&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/CHANGES.txt (original)
+++ hadoop/common/branches/yahoo-merge/CHANGES.txt Mon May 23 21:46:49 2011
@@ -27,6 +27,8 @@ Trunk (unreleased changes)
 
     HADOOP-7202. Improve shell Command base class.  (Daryn Sharp via szetszwo)
 
+    HADOOP-7224. Add CommandFactory to shell.  (Daryn Sharp via szetszwo)
+
   OPTIMIZATIONS
 
   BUG FIXES

Propchange: hadoop/common/branches/yahoo-merge/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 21:46:49 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/CHANGES.txt:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1092565,1092832,1100026
+/hadoop/common/trunk/CHANGES.txt:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092565,1092832,1100026
 /hadoop/core/branches/branch-0.18/CHANGES.txt:727226
 /hadoop/core/branches/branch-0.19/CHANGES.txt:713112
 /hadoop/core/trunk/CHANGES.txt:776175-785643,785929-786278

Propchange: hadoop/common/branches/yahoo-merge/src/contrib/ec2/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 21:46:49 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/src/contrib/ec2:1043117,1078148,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1092565,1100026
+/hadoop/common/trunk/src/contrib/ec2:1043117,1078148,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092565,1100026
 /hadoop/core/branches/branch-0.19/core/src/contrib/ec2:713112
 /hadoop/core/trunk/src/contrib/ec2:776175-784663

Propchange: hadoop/common/branches/yahoo-merge/src/docs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 21:46:49 2011
@@ -1,2 +1,2 @@
-/hadoop/common/trunk/src/docs:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1092565,1100026
+/hadoop/common/trunk/src/docs:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092565,1100026
 /hadoop/core/branches/branch-0.19/src/docs:713112

Propchange: hadoop/common/branches/yahoo-merge/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 21:46:49 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/src/java:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1092565,1100026
+/hadoop/common/trunk/src/java:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092565,1100026
 /hadoop/core/branches/branch-0.19/core/src/java:713112
 /hadoop/core/trunk/src/core:776175-785643,785929-786278

Modified: hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java?rev=1126769&r1=1126768&r2=1126769&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java (original)
+++ hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java Mon May
23 21:46:49 2011
@@ -26,7 +26,6 @@ import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.TimeZone;
 import java.util.zip.GZIPInputStream;
@@ -34,8 +33,10 @@ import java.util.zip.GZIPInputStream;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.shell.Command;
+import org.apache.hadoop.fs.shell.CommandFactory;
 import org.apache.hadoop.fs.shell.CommandFormat;
-import org.apache.hadoop.fs.shell.Count;
+import org.apache.hadoop.fs.shell.FsCommand;
 import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.io.IOUtils;
@@ -57,6 +58,8 @@ public class FsShell extends Configured 
 
   protected FileSystem fs;
   private Trash trash;
+  protected CommandFactory commandFactory;
+
   public static final SimpleDateFormat dateForm = 
     new SimpleDateFormat("yyyy-MM-dd HH:mm");
   protected static final SimpleDateFormat modifFmt =
@@ -82,6 +85,7 @@ public class FsShell extends Configured 
     super(conf);
     fs = null;
     trash = null;
+    commandFactory = new CommandFactory();
   }
   
   protected void init() throws IOException {
@@ -1385,9 +1389,7 @@ public class FsShell extends Configured 
       "[-tail [-f] <path>] [-text <path>]\n\t" +
       "[" + FsShellPermissions.CHMOD_USAGE + "]\n\t" +
       "[" + FsShellPermissions.CHOWN_USAGE + "]\n\t" +
-      "[" + FsShellPermissions.CHGRP_USAGE + "]\n\t" +      
-      "[" + Count.USAGE + "]\n\t" +      
-      "[-help [cmd]]\n";
+      "[" + FsShellPermissions.CHGRP_USAGE + "]";
 
     String conf ="-conf <configuration file>:  Specify an application configuration
file.";
  
@@ -1546,7 +1548,10 @@ public class FsShell extends Configured 
     String help = "-help [cmd]: \tDisplays help for given command or all commands if none\n"
+
       "\t\tis specified.\n";
 
-    if ("fs".equals(cmd)) {
+    Command instance = commandFactory.getInstance("-" + cmd);
+    if (instance != null) {
+      System.out.println(instance.getDescription());
+    } else if ("fs".equals(cmd)) {
       System.out.println(fs);
     } else if ("conf".equals(cmd)) {
       System.out.println(conf);
@@ -1610,12 +1615,16 @@ public class FsShell extends Configured 
       System.out.println(chown);
     } else if ("chgrp".equals(cmd)) {
       System.out.println(chgrp);
-    } else if (Count.NAME.equals(cmd)) {
-      System.out.println(Count.DESCRIPTION);
     } else if ("help".equals(cmd)) {
       System.out.println(help);
     } else {
       System.out.println(summary);
+      for (String thisCmdName : commandFactory.getNames()) {
+        instance = commandFactory.getInstance(thisCmdName);
+        System.out.println(instance.getUsage());
+      }
+      System.out.println("\t[-help [cmd]]\n");
+      
       System.out.println(fs);
       System.out.println(ls);
       System.out.println(lsr);
@@ -1644,7 +1653,12 @@ public class FsShell extends Configured 
       System.out.println(chmod);
       System.out.println(chown);      
       System.out.println(chgrp);
-      System.out.println(Count.DESCRIPTION);
+
+      for (String thisCmdName : commandFactory.getNames()) {
+        instance = commandFactory.getInstance(thisCmdName);
+        System.out.println(instance.getDescription());
+      }
+
       System.out.println(help);
     }        
   }
@@ -1727,9 +1741,13 @@ public class FsShell extends Configured 
    * Displays format of commands.
    * 
    */
-  private static void printUsage(String cmd) {
+  private void printUsage(String cmd) {
     String prefix = "Usage: java " + FsShell.class.getSimpleName();
-    if ("-fs".equals(cmd)) {
+
+    Command instance = commandFactory.getInstance(cmd);
+    if (instance != null) {
+      System.err.println(prefix + " [" + instance.getUsage() + "]");
+    } else if ("-fs".equals(cmd)) {
       System.err.println("Usage: java FsShell" + 
                          " [-fs <local | file system URI>]");
     } else if ("-conf".equals(cmd)) {
@@ -1747,8 +1765,6 @@ public class FsShell extends Configured 
     } else if ("-df".equals(cmd) ) {
       System.err.println("Usage: java FsShell" +
                          " [" + cmd + " [<path>]]");
-    } else if ("-count".equals(cmd)) {
-      System.err.println(prefix + " [" + Count.USAGE + "]");
     } else if ("-rm".equals(cmd) || "-rmr".equals(cmd)) {
       System.err.println("Usage: java FsShell [" + cmd + 
                            " [-skipTrash] <src>]");
@@ -1786,7 +1802,6 @@ public class FsShell extends Configured 
       System.err.println("           [-df [<path>]]");
       System.err.println("           [-du [-s] [-h] <path>]");
       System.err.println("           [-dus <path>]");
-      System.err.println("           [" + Count.USAGE + "]");
       System.err.println("           [-mv <src> <dst>]");
       System.err.println("           [-cp <src> <dst>]");
       System.err.println("           [-rm [-skipTrash] <path>]");
@@ -1810,6 +1825,10 @@ public class FsShell extends Configured 
       System.err.println("           [" + FsShellPermissions.CHMOD_USAGE + "]");      
       System.err.println("           [" + FsShellPermissions.CHOWN_USAGE + "]");
       System.err.println("           [" + FsShellPermissions.CHGRP_USAGE + "]");
+      for (String name : commandFactory.getNames()) {
+      	instance = commandFactory.getInstance(name);
+        System.err.println("           [" + instance.getUsage() + "]");
+      }
       System.err.println("           [-help [cmd]]");
       System.err.println();
       ToolRunner.printGenericCommandUsage(System.err);
@@ -1820,7 +1839,12 @@ public class FsShell extends Configured 
    * run
    */
   public int run(String argv[]) throws Exception {
-
+    // TODO: This isn't the best place, but this class is being abused with
+    // subclasses which of course override this method.  There really needs
+    // to be a better base class for all commands
+    commandFactory.setConf(getConf());
+    commandFactory.registerCommands(FsCommand.class);
+    
     if (argv.length < 1) {
       printUsage(""); 
       return -1;
@@ -1873,7 +1897,19 @@ public class FsShell extends Configured 
 
     exitCode = 0;
     try {
-      if ("-put".equals(cmd) || "-copyFromLocal".equals(cmd)) {
+      Command instance = commandFactory.getInstance(cmd);
+      if (instance != null) {
+        try {
+          exitCode = instance.run(Arrays.copyOfRange(argv, i, argv.length));
+        } catch (Exception e) {
+          exitCode = -1;
+          LOG.debug("Error", e);
+          instance.displayError(e);
+          if (e instanceof IllegalArgumentException) {
+            printUsage(cmd);
+          }
+        }
+      } else if ("-put".equals(cmd) || "-copyFromLocal".equals(cmd)) {
         Path[] srcs = new Path[argv.length-2];
         for (int j=0 ; i < argv.length-1 ;) 
           srcs[j++] = new Path(argv[i++]);
@@ -1934,12 +1970,6 @@ public class FsShell extends Configured 
         du(argv, i);
       } else if ("-dus".equals(cmd)) {
         dus(argv, i);
-      } else if ("-count".equals(cmd)) {
-        // TODO: next two lines are a temporary crutch until this entire
-        // block is overhauled
-        Count runner = ReflectionUtils.newInstance(Count.class, getConf());
-        runner.setCommandName(cmd); // TODO: will change with factory
-        exitCode = runner.run(Arrays.copyOfRange(argv, 1, argv.length));
       } else if ("-mkdir".equals(cmd)) {
         exitCode = doall(cmd, argv, i);
       } else if ("-touchz".equals(cmd)) {

Modified: hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/Command.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/Command.java?rev=1126769&r1=1126768&r2=1126769&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/Command.java (original)
+++ hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/Command.java Mon
May 23 21:46:49 2011
@@ -365,4 +365,35 @@ abstract public class Command extends Co
   public void displayWarning(String message) {
     err.println(getCommandName() + ": " + message);
   }
+  
+  /**
+   * The short usage suitable for the synopsis
+   * @return "name options"
+   */
+  public String getUsage() {
+    return getCommandField("USAGE");
+  }
+
+  /**
+   * The long usage suitable for help output
+   * @return text of the usage
+   */
+  public String getDescription() {
+    return getCommandField("DESCRIPTION");
+  }
+
+  /**
+   * Get a public static class field
+   * @param field the field to retrieve
+   * @return String of the field
+   */
+  private String getCommandField(String field) {
+    String value;
+    try {
+      value = (String)this.getClass().getField(field).get(null);
+    } catch (Exception e) {
+      throw new RuntimeException(StringUtils.stringifyException(e));
+    }
+    return value;
+  }
 }

Modified: hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/Count.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/Count.java?rev=1126769&r1=1126768&r2=1126769&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/Count.java (original)
+++ hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/Count.java Mon
May 23 21:46:49 2011
@@ -34,6 +34,14 @@ import org.apache.hadoop.fs.FsShell;
 @InterfaceStability.Evolving
 
 public class Count extends FsCommand {
+  /**
+   * Register the names for the count command
+   * @param factory the command factory that will instantiate this class
+   */
+  public static void registerCommands(CommandFactory factory) {
+    factory.addClass(Count.class, "-count");
+  }
+
   public static final String NAME = "count";
   public static final String USAGE = "-" + NAME + "[-q] <path>";
   public static final String DESCRIPTION = CommandUtils.formatDescription(USAGE, 

Modified: hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java?rev=1126769&r1=1126768&r2=1126769&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java
(original)
+++ hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java
Mon May 23 21:46:49 2011
@@ -37,6 +37,14 @@ import org.apache.hadoop.fs.Path;
 // used to implement unnecessary abstract methods in the base class
 
 abstract public class FsCommand extends Command {
+  /**
+   * Register the command classes used by the fs subcommand
+   * @param factory where to register the class
+   */
+  public static void registerCommands(CommandFactory factory) {
+    Count.registerCommands(factory);
+  }
+
   protected FsCommand() {}
   
   protected FsCommand(Configuration conf) {

Propchange: hadoop/common/branches/yahoo-merge/src/test/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 21:46:49 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/src/test/core:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1092565,1100026
+/hadoop/common/trunk/src/test/core:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092565,1100026
 /hadoop/core/branches/branch-0.19/core/src/test/core:713112
 /hadoop/core/trunk/src/test/core:776175-785643,785929-786278



Mime
View raw message