hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maha...@apache.org
Subject svn commit: r686840 - in /hadoop/core/trunk: ./ src/core/org/apache/hadoop/fs/ src/core/org/apache/hadoop/fs/shell/ src/hdfs/org/apache/hadoop/hdfs/tools/ src/test/org/apache/hadoop/hdfs/
Date Mon, 18 Aug 2008 18:13:18 GMT
Author: mahadev
Date: Mon Aug 18 11:13:17 2008
New Revision: 686840

URL: http://svn.apache.org/viewvc?rev=686840&view=rev
Log:
HADOOP-3962. Shell command fs -count should support paths with different file systems. (Tsz
Wo (Nicholas), SZE via mahadev)

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/core/org/apache/hadoop/fs/FsShell.java
    hadoop/core/trunk/src/core/org/apache/hadoop/fs/shell/Command.java
    hadoop/core/trunk/src/core/org/apache/hadoop/fs/shell/Count.java
    hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java
    hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestDFSShell.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=686840&r1=686839&r2=686840&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Aug 18 11:13:17 2008
@@ -329,6 +329,9 @@
     HADOOP-3933. DataNode sometimes sends up to io.byte.per.checksum bytes 
     more than required to client. (Ning Li via rangadi)
 
+    HADOOP-3962. Shell command "fs -count" should support paths with different
+    file systems. (Tsz Wo (Nicholas), SZE via mahadev)
+
 Release 0.18.0 - 2008-08-19
 
   INCOMPATIBLE CHANGES

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/FsShell.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/FsShell.java?rev=686840&r1=686839&r2=686840&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/FsShell.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/FsShell.java Mon Aug 18 11:13:17 2008
@@ -1522,7 +1522,7 @@
         } else if ("-dus".equals(cmd)) {
           dus(argv[i]);
         } else if (Count.matches(cmd)) {
-          new Count(argv, i, fs).runAll();
+          new Count(argv, i, getConf()).runAll();
         } else if ("-ls".equals(cmd)) {
           ls(argv[i], false);
         } else if ("-lsr".equals(cmd)) {
@@ -1772,7 +1772,7 @@
           dus(".");
         }         
       } else if (Count.matches(cmd)) {
-        exitCode = new Count(argv, i, fs).runAll();
+        exitCode = new Count(argv, i, getConf()).runAll();
       } else if ("-mkdir".equals(cmd)) {
         exitCode = doall(cmd, argv, i);
       } else if ("-touchz".equals(cmd)) {

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/shell/Command.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/shell/Command.java?rev=686840&r1=686839&r2=686840&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/shell/Command.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/shell/Command.java Mon Aug 18 11:13:17
2008
@@ -19,6 +19,8 @@
 
 import java.io.*;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -27,13 +29,12 @@
 /**
  * An abstract class for the execution of a file system command
  */
-abstract public class Command {
-  final protected FileSystem fs;
+abstract public class Command extends Configured {
   protected String[] args;
   
   /** Constructor */
-  protected Command(FileSystem fs) {
-    this.fs = fs;
+  protected Command(Configuration conf) {
+    super(conf);
   }
   
   /** Return the command's name excluding the leading character - */
@@ -57,6 +58,7 @@
     for (String src : args) {
       try {
         Path srcPath = new Path(src);
+        FileSystem fs = srcPath.getFileSystem(getConf());
         FileStatus[] statuses = fs.globStatus(srcPath);
         if (statuses == null) {
           System.err.println("Can not find listing for " + src);

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/shell/Count.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/shell/Count.java?rev=686840&r1=686839&r2=686840&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/shell/Count.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/shell/Count.java Mon Aug 18 11:13:17 2008
@@ -20,6 +20,7 @@
 import java.io.*;
 import java.util.List;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
@@ -41,10 +42,9 @@
    * 
    * @param cmd the count command
    * @param pos the starting index of the arguments 
-   * @param fs the file system handler
    */
-  public Count(String[] cmd, int pos, FileSystem fs) {
-    super(fs);
+  public Count(String[] cmd, int pos, Configuration conf) {
+    super(conf);
     CommandFormat c = new CommandFormat(NAME, 1, Integer.MAX_VALUE, "q");
     List<String> parameters = c.parse(cmd, pos);
     this.args = parameters.toArray(new String[parameters.size()]);
@@ -70,6 +70,7 @@
 
   @Override
   protected void run(Path path) throws IOException {
+    FileSystem fs = path.getFileSystem(getConf());
     System.out.println(fs.getContentSummary(path).toString(qOption) + path);
   }
 }

Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java?rev=686840&r1=686839&r2=686840&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java Mon Aug 18 11:13:17
2008
@@ -47,13 +47,15 @@
    * An abstract class for the execution of a file system command
    */
   abstract private static class DFSAdminCommand extends Command {
+    final DistributedFileSystem dfs;
     /** Constructor */
     public DFSAdminCommand(FileSystem fs) {
-      super(fs);
+      super(fs.getConf());
       if (!(fs instanceof DistributedFileSystem)) {
         throw new IllegalArgumentException("FileSystem " + fs.getUri() + 
             " is not a distributed file system");
       }
+      this.dfs = (DistributedFileSystem)fs;
     }
   }
   
@@ -92,7 +94,7 @@
 
     @Override
     public void run(Path path) throws IOException {
-      ((DistributedFileSystem)fs).clearQuota(path);
+      dfs.clearQuota(path);
     }
   }
   
@@ -139,7 +141,7 @@
 
     @Override
     public void run(Path path) throws IOException {
-      ((DistributedFileSystem)fs).setQuota(path, quota);
+      dfs.setQuota(path, quota);
     }
   }
   

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestDFSShell.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestDFSShell.java?rev=686840&r1=686839&r2=686840&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestDFSShell.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestDFSShell.java Mon Aug 18 11:13:17
2008
@@ -673,10 +673,20 @@
       String root = createTree(dfs, "count");
 
       // Verify the counts
-      runCount(root, 2, 4, dfs);
-      runCount(root + "2", 2, 1, dfs);
-      runCount(root + "2/f1", 0, 1, dfs);
-      runCount(root + "2/sub", 1, 0, dfs);
+      runCount(root, 2, 4, conf);
+      runCount(root + "2", 2, 1, conf);
+      runCount(root + "2/f1", 0, 1, conf);
+      runCount(root + "2/sub", 1, 0, conf);
+
+      final FileSystem localfs = FileSystem.getLocal(conf);
+      Path localpath = new Path(TEST_ROOT_DIR, "testcount");
+      localpath = localpath.makeQualified(localfs);
+      localfs.mkdirs(localpath);
+      
+      final String localstr = localpath.toString();
+      System.out.println("localstr=" + localstr);
+      runCount(localstr, 1, 0, conf);
+      assertEquals(0, new Count(new String[]{root, localstr}, 0, conf).runAll());
     } finally {
       try {
         dfs.close();
@@ -685,17 +695,17 @@
       cluster.shutdown();
     }
   }
-  private void runCount(String path, long dirs, long files, FileSystem fs
+  private void runCount(String path, long dirs, long files, Configuration conf
     ) throws IOException {
     ByteArrayOutputStream bytes = new ByteArrayOutputStream(); 
     PrintStream out = new PrintStream(bytes);
     PrintStream oldOut = System.out;
     System.setOut(out);
     Scanner in = null;
+    String results = null;
     try {
-      new Count(new String[]{path}, 0, fs).runAll();
-      String results = bytes.toString();
-      System.out.println(results);
+      new Count(new String[]{path}, 0, conf).runAll();
+      results = bytes.toString();
       in = new Scanner(results);
       assertEquals(dirs, in.nextLong());
       assertEquals(files, in.nextLong());
@@ -703,6 +713,7 @@
       if (in!=null) in.close();
       IOUtils.closeStream(out);
       System.setOut(oldOut);
+      System.out.println("results:\n" + results);
     }
   }
 



Mime
View raw message