hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r641681 - in /hadoop/core/trunk: ./ src/docs/src/documentation/content/xdocs/ src/java/org/apache/hadoop/fs/ src/test/org/apache/hadoop/dfs/
Date Thu, 27 Mar 2008 01:23:01 GMT
Author: cdouglas
Date: Wed Mar 26 18:22:59 2008
New Revision: 641681

URL: http://svn.apache.org/viewvc?rev=641681&view=rev
Log:
HADOOP-3091. Modify FsShell command -put to accept multiple sources.
Contributed by Lohit Vijaya Renu.


Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_shell.xml
    hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
    hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileUtil.java
    hadoop/core/trunk/src/java/org/apache/hadoop/fs/FsShell.java
    hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDFSShell.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=641681&r1=641680&r2=641681&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed Mar 26 18:22:59 2008
@@ -153,6 +153,9 @@
     HADOOP-2796. Enables distinguishing exit codes from user code vis-a-vis
     HOD's exit code. (Hemanth Yamijala via ddas)
 
+    HADOOP-3091. Modify FsShell command -put to accept multiple sources.
+    (Lohit Vijaya Renu via cdouglas)
+
   OPTIMIZATIONS
 
     HADOOP-2790.  Fixed inefficient method hasSpeculativeTask by removing

Modified: hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_shell.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_shell.xml?rev=641681&r1=641680&r2=641681&view=diff
==============================================================================
--- hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_shell.xml (original)
+++ hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_shell.xml Wed Mar 26 18:22:59
2008
@@ -249,13 +249,16 @@
 		<section>
 			<title id="putlink"> put </title>
 			<p>
-				<code>Usage: hadoop dfs -put &lt;localsrc&gt; &lt;dst&gt;</code>
+				<code>Usage: hadoop dfs -put &lt;localsrc&gt; ... &lt;dst&gt;</code>
 			</p>
-			<p>Copy src from local file system to the destination filesystem. Also reads input
from stdin and writes to destination filesystem.<br/>
+			<p>Copy single src, or multiple srcs from local file system to the destination filesystem.
Also reads input from stdin and writes to destination filesystem.<br/>
 	   </p>
 			<ul>
 				<li>
 					<code> hadoop dfs -put localfile /user/hadoop/hadoopfile</code>
+				</li>
+				<li>
+					<code> hadoop dfs -put localfile1 localfile2 /user/hadoop/hadoopdir</code>
 				</li>
 				<li>
 					<code> hadoop dfs -put localfile hdfs://host:port/hadoop/hadoopfile</code>

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java?rev=641681&r1=641680&r2=641681&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java Wed Mar 26 18:22:59 2008
@@ -1104,6 +1104,15 @@
   }
 
   /**
+   * The src files is on the local disk.  Add it to FS at
+   * the given dst name, removing the source afterwards.
+   */
+  public void moveFromLocalFile(Path[] srcs, Path dst)
+    throws IOException {
+    copyFromLocalFile(true, true, srcs, dst);
+  }
+
+  /**
    * The src file is on the local disk.  Add it to FS at
    * the given dst name, removing the source afterwards.
    */
@@ -1121,7 +1130,19 @@
     throws IOException {
     copyFromLocalFile(delSrc, true, src, dst);
   }
-
+  
+  /**
+   * The src files are on the local disk.  Add it to FS at
+   * the given dst name.
+   * delSrc indicates if the source should be removed
+   */
+  public void copyFromLocalFile(boolean delSrc, boolean overwrite, 
+                                Path[] srcs, Path dst)
+    throws IOException {
+    Configuration conf = getConf();
+    FileUtil.copy(getLocal(conf), srcs, this, dst, delSrc, overwrite, conf);
+  }
+  
   /**
    * The src file is on the local disk.  Add it to FS at
    * the given dst name.

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileUtil.java?rev=641681&r1=641680&r2=641681&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileUtil.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileUtil.java Wed Mar 26 18:22:59 2008
@@ -132,7 +132,46 @@
                              Configuration conf) throws IOException {
     return copy(srcFS, src, dstFS, dst, deleteSource, true, conf);
   }
-  
+
+  public static boolean copy(FileSystem srcFS, Path[] srcs, 
+                             FileSystem dstFS, Path dst,
+                             boolean deleteSource, 
+                             boolean overwrite, Configuration conf)
+                             throws IOException {
+    boolean gotException = false;
+    boolean returnVal = true;
+    StringBuffer exceptions = new StringBuffer();
+
+    if (srcs.length == 1)
+      return copy(srcFS, srcs[0], dstFS, dst, deleteSource, overwrite, conf);
+
+    // Check if dest is directory
+    if (!dstFS.exists(dst)) {
+      throw new IOException("`" + dst +"': specified destination directory " +
+                            "doest not exist");
+    } else {
+      FileStatus sdst = dstFS.getFileStatus(dst);
+      if (!sdst.isDir()) 
+        throw new IOException("copying multiple files, but last argument `" +
+                              dst + "' is not a directory");
+    }
+
+    for (Path src : srcs) {
+      try {
+        if (!copy(srcFS, src, dstFS, dst, deleteSource, overwrite, conf))
+          returnVal = false;
+      } catch (IOException e) {
+        gotException = true;
+        exceptions.append(e.getMessage());
+        exceptions.append("\n");
+      }
+    }
+    if (gotException) {
+      throw new IOException(exceptions.toString());
+    }
+    return returnVal;
+  }
+
   /** Copy files between FileSystems. */
   public static boolean copy(FileSystem srcFS, Path src, 
                              FileSystem dstFS, Path dst, 
@@ -204,7 +243,7 @@
       return true;
     }
   }  
-
+  
   /** Copy local files to a FileSystem. */
   public static boolean copy(File src,
                              FileSystem dstFS, Path dst,

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/FsShell.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/FsShell.java?rev=641681&r1=641680&r2=641681&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/FsShell.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/FsShell.java Wed Mar 26 18:22:59 2008
@@ -121,26 +121,43 @@
     }
   }
 
+  
   /**
-   * Add a local file to the indicated FileSystem name. src is kept.
+   * Add local files to the indicated FileSystem name. src is kept.
    */
-  void copyFromLocal(Path src, String dstf) throws IOException {
+  void copyFromLocal(Path[] srcs, String dstf) throws IOException {
     Path dstPath = new Path(dstf);
     FileSystem dstFs = dstPath.getFileSystem(getConf());
+    dstFs.copyFromLocalFile(false, false, srcs, dstPath);
+  }
+  
+  /**
+   * Add a local file to the indicated FileSystem name. src is kept.
+   */
+  void copyFromLocal(Path src, String dstf) throws IOException {
     if (src.toString().equals("-")) {
+      Path dstPath = new Path(dstf);
+      FileSystem dstFs = dstPath.getFileSystem(getConf());
       copyFromStdin(dstPath, dstFs);
     } else {
-      dstFs.copyFromLocalFile(false, false, src, new Path(dstf));
+      copyFromLocal(new Path[]{src}, dstf);
     }
   }
 
   /**
-   * Add a local file to the indicated FileSystem name. src is removed.
+   * Add local files to the indicated FileSystem name. src is removed.
    */
-  void moveFromLocal(Path src, String dstf) throws IOException {
+  void moveFromLocal(Path[] srcs, String dstf) throws IOException {
     Path dstPath = new Path(dstf);
     FileSystem dstFs = dstPath.getFileSystem(getConf());
-    dstFs.moveFromLocalFile(src, dstPath);
+    dstFs.moveFromLocalFile(srcs, dstPath);
+  }
+
+  /**
+   * Add a local file to the indicated FileSystem name. src is removed.
+   */
+  void moveFromLocal(Path src, String dstf) throws IOException {
+    moveFromLocal((new Path[]{src}), dstf);
   }
 
   /**
@@ -1248,8 +1265,9 @@
       "hadoop fs [-fs <local | file system URI>] [-conf <configuration file>]\n\t"
+
       "[-D <property=value>] [-ls <path>] [-lsr <path>] [-du <path>]\n\t"
+ 
       "[-dus <path>] [-mv <src> <dst>] [-cp <src> <dst>] [-rm
<src>]\n\t" + 
-      "[-rmr <src>] [-put <localsrc> <dst>] [-copyFromLocal <localsrc>
<dst>]\n\t" +
-      "[-moveFromLocal <localsrc> <dst>] [" + GET_SHORT_USAGE + "\n\t" +
+      "[-rmr <src>] [-put <localsrc> ... <dst>] [-copyFromLocal <localsrc>
... <dst>]\n\t" +
+      "[-moveFromLocal <localsrc> ... <dst>] [" + 
+      GET_SHORT_USAGE + "\n\t" +
       "[-getmerge <src> <localdst> [addnl]] [-cat <src>]\n\t" +
       "[" + COPYTOLOCAL_SHORT_USAGE + "] [-moveToLocal <src> <localdst>]\n\t"
+
       "[-mkdir <path>] [-report] [" + SETREP_SHORT_USAGE + "]\n\t" +
@@ -1318,13 +1336,14 @@
     String rmr = "-rmr <src>: \tRemove all directories which match the specified file
\n" +
       "\t\tpattern. Equivlent to the Unix command \"rm -rf <src>\"\n";
 
-    String put = "-put <localsrc> <dst>: \tCopy a single file from the local
file system \n" +
-      "\t\tinto fs. \n";
+    String put = "-put <localsrc> ... <dst>: \tCopy files " + 
+    "from the local file system \n\t\tinto fs. \n";
 
-    String copyFromLocal = "-copyFromLocal <localsrc> <dst>:  Identical to the
-put command.\n";
+    String copyFromLocal = "-copyFromLocal <localsrc> ... <dst>:" +
+    " Identical to the -put command.\n";
 
-    String moveFromLocal = "-moveFromLocal <localsrc> <dst>:  Same as -put, except
that the source is\n" +
-      "\t\tdeleted after it's copied.\n"; 
+    String moveFromLocal = "-moveFromLocal <localsrc> ... <dst>:" +
+    " Same as -put, except that the source is\n\t\tdeleted after it's copied.\n"; 
 
     String get = GET_SHORT_USAGE
       + ":  Copy files that match the file pattern <src> \n" +
@@ -1618,9 +1637,9 @@
       System.err.println("           [-rm <path>]");
       System.err.println("           [-rmr <path>]");
       System.err.println("           [-expunge]");
-      System.err.println("           [-put <localsrc> <dst>]");
-      System.err.println("           [-copyFromLocal <localsrc> <dst>]");
-      System.err.println("           [-moveFromLocal <localsrc> <dst>]");
+      System.err.println("           [-put <localsrc> ... <dst>]");
+      System.err.println("           [-copyFromLocal <localsrc> ... <dst>]");
+      System.err.println("           [-moveFromLocal <localsrc> ... <dst>]");
       System.err.println("           [" + GET_SHORT_USAGE + "]");
       System.err.println("           [-getmerge <src> <localdst> [addnl]]");
       System.err.println("           [-cat <src>]");
@@ -1661,7 +1680,7 @@
     //
     if ("-put".equals(cmd) || "-test".equals(cmd) ||
         "-copyFromLocal".equals(cmd) || "-moveFromLocal".equals(cmd)) {
-      if (argv.length != 3) {
+      if (argv.length < 3) {
         printUsage(cmd);
         return exitCode;
       }
@@ -1701,9 +1720,15 @@
     exitCode = 0;
     try {
       if ("-put".equals(cmd) || "-copyFromLocal".equals(cmd)) {
-        copyFromLocal(new Path(argv[i++]), argv[i++]);
+        Path[] srcs = new Path[argv.length-2];
+        for (int j=0 ; i < argv.length-1 ;) 
+          srcs[j++] = new Path(argv[i++]);
+        copyFromLocal(srcs, argv[i++]);
       } else if ("-moveFromLocal".equals(cmd)) {
-        moveFromLocal(new Path(argv[i++]), argv[i++]);
+        Path[] srcs = new Path[argv.length-2];
+        for (int j=0 ; i < argv.length-1 ;) 
+          srcs[j++] = new Path(argv[i++]);
+        moveFromLocal(srcs, argv[i++]);
       } else if ("-get".equals(cmd) || "-copyToLocal".equals(cmd)) {
         copyToLocal(argv, i);
       } else if ("-getmerge".equals(cmd)) {

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDFSShell.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDFSShell.java?rev=641681&r1=641680&r2=641681&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDFSShell.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDFSShell.java Wed Mar 26 18:22:59
2008
@@ -194,6 +194,30 @@
   
       try {copy2ndFileThread.join();} catch (InterruptedException e) { }
       System.setSecurityManager(sm);
+
+      // copy multiple files to destination directory
+      final Path destmultiple = mkdir(dfs, new Path("/test/putmultiple"));
+      Path[] srcs = new Path[2];
+      srcs[0] = new Path(f1.getPath());
+      srcs[1] = new Path(f2.getPath());
+      dfs.copyFromLocalFile(false, false, srcs, destmultiple);
+      srcs[0] = new Path(destmultiple,"f1"); 
+      srcs[1] = new Path(destmultiple,"f2"); 
+      assertTrue(dfs.exists(srcs[0]));
+      assertTrue(dfs.exists(srcs[1]));
+
+      // move multiple files to destination directory
+      final Path destmultiple2 = mkdir(dfs, new Path("/test/movemultiple"));
+      srcs[0] = new Path(f1.getPath());
+      srcs[1] = new Path(f2.getPath());
+      dfs.moveFromLocalFile(srcs, destmultiple2);
+      assertFalse(dfs.exists(srcs[0]));
+      assertFalse(dfs.exists(srcs[1]));
+      srcs[0] = new Path(destmultiple2, "f1");
+      srcs[1] = new Path(destmultiple2, "f2");
+      assertTrue(dfs.exists(srcs[0]));
+      assertTrue(dfs.exists(srcs[1]));
+
       f1.delete();
       f2.delete();
     } finally {



Mime
View raw message