hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject git commit: HDFS-6928. 'hdfs put' command should accept lazyPersist flag for testing. (Arpit Agarwal)
Date Thu, 28 Aug 2014 22:53:45 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-6581 722cc92fa -> bbaa7dc28


HDFS-6928. 'hdfs put' command should accept lazyPersist flag for testing. (Arpit Agarwal)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bbaa7dc2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bbaa7dc2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bbaa7dc2

Branch: refs/heads/HDFS-6581
Commit: bbaa7dc28db75d9b3700c6ff95222d8e1de29c15
Parents: 722cc92
Author: arp <arp@apache.org>
Authored: Thu Aug 28 15:53:12 2014 -0700
Committer: arp <arp@apache.org>
Committed: Thu Aug 28 15:53:12 2014 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/fs/FilterFileSystem.java  | 13 ++++++-
 .../hadoop/fs/shell/CommandWithDestination.java | 32 ++++++++++++++---
 .../apache/hadoop/fs/shell/CopyCommands.java    | 16 +++++----
 .../java/org/apache/hadoop/fs/shell/Stat.java   |  7 +++-
 .../src/test/resources/testConf.xml             | 28 +++++++++++----
 .../hadoop-hdfs/CHANGES-HDFS-6581.txt           |  2 ++
 .../src/test/resources/testHDFSConf.xml         | 36 ++++++++++++++++++++
 7 files changed, 115 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbaa7dc2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
index 52706f4..e729e67 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
@@ -179,8 +179,19 @@ public class FilterFileSystem extends FileSystem {
     return fs.create(f, permission,
         overwrite, bufferSize, replication, blockSize, progress);
   }
-  
 
+  @Override
+  public FSDataOutputStream create(Path f,
+        FsPermission permission,
+        EnumSet<CreateFlag> flags,
+        int bufferSize,
+        short replication,
+        long blockSize,
+        Progressable progress,
+        ChecksumOpt checksumOpt) throws IOException {
+    return fs.create(f, permission,
+      flags, bufferSize, replication, blockSize, progress);
+  }
   
   @Override
   @Deprecated

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbaa7dc2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
index da67f1c..5d0d9d6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
@@ -30,6 +30,7 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.NoSuchElementException;
 
+import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FilterFileSystem;
@@ -45,6 +46,9 @@ import org.apache.hadoop.fs.permission.AclUtil;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.IOUtils;
 
+import static org.apache.hadoop.fs.CreateFlag.CREATE;
+import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST;
+
 /**
  * Provides: argument processing to ensure the destination is valid
  * for the number of source arguments.  A processPaths that accepts both
@@ -56,6 +60,7 @@ abstract class CommandWithDestination extends FsCommand {
   private boolean overwrite = false;
   private boolean verifyChecksum = true;
   private boolean writeChecksum = true;
+  private boolean lazyPersist = false;
   
   /**
    * The name of the raw xattr namespace. It would be nice to use
@@ -78,6 +83,10 @@ abstract class CommandWithDestination extends FsCommand {
     overwrite = flag;
   }
   
+  protected void setLazyPersist(boolean flag) {
+    lazyPersist = flag;
+  }
+
   protected void setVerifyChecksum(boolean flag) {
     verifyChecksum = flag;
   }
@@ -379,7 +388,7 @@ abstract class CommandWithDestination extends FsCommand {
     try {
       PathData tempTarget = target.suffix("._COPYING_");
       targetFs.setWriteChecksum(writeChecksum);
-      targetFs.writeStreamToFile(in, tempTarget);
+      targetFs.writeStreamToFile(in, tempTarget, lazyPersist);
       targetFs.rename(tempTarget, target);
     } finally {
       targetFs.close(); // last ditch effort to ensure temp file is removed
@@ -449,10 +458,11 @@ abstract class CommandWithDestination extends FsCommand {
       super(fs);
     }
 
-    void writeStreamToFile(InputStream in, PathData target) throws IOException {
+    void writeStreamToFile(InputStream in, PathData target,
+                           boolean lazyPersist) throws IOException {
       FSDataOutputStream out = null;
       try {
-        out = create(target);
+        out = create(target, lazyPersist);
         IOUtils.copyBytes(in, out, getConf(), true);
       } finally {
         IOUtils.closeStream(out); // just in case copyBytes didn't
@@ -460,9 +470,21 @@ abstract class CommandWithDestination extends FsCommand {
     }
     
     // tag created files as temp files
-    FSDataOutputStream create(PathData item) throws IOException {
+    FSDataOutputStream create(PathData item, boolean lazyPersist)
+        throws IOException {
       try {
-        return create(item.path, true);
+        EnumSet<CreateFlag> createFlags = EnumSet.of(CREATE);
+        if (lazyPersist) {
+          createFlags.add(LAZY_PERSIST);
+        }
+        return create(item.path,
+                      null,
+                      createFlags,
+                      getConf().getInt("io.file.buffer.size", 4096),
+                      lazyPersist ? 1 : getDefaultReplication(item.path),
+                      getDefaultBlockSize(),
+                      null,
+                      null);
       } finally { // might have been created but stream was interrupted
         deleteOnExit(item.path);
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbaa7dc2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
index 3fd870c..afd1115 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
@@ -215,21 +215,25 @@ class CopyCommands {
    */
   public static class Put extends CommandWithDestination {
     public static final String NAME = "put";
-    public static final String USAGE = "[-f] [-p] <localsrc> ... <dst>";
+    public static final String USAGE = "[-f] [-p] [-l] <localsrc> ... <dst>";
     public static final String DESCRIPTION =
       "Copy files from the local file system " +
       "into fs. Copying fails if the file already " +
-      "exists, unless the -f flag is given. Passing " +
-      "-p preserves access and modification times, " +
-      "ownership and the mode. Passing -f overwrites " +
-      "the destination if it already exists.\n";
+      "exists, unless the -f flag is given.\n" +
+      "Flags:\n" +
+      "  -p : Preserves access and modification times, ownership and the mode.\n" +
+      "  -f : Overwrites the destination if it already exists.\n" +
+      "  -l : Allow DataNode to lazily persist the file to disk. Forces\n" +
+      "       replication factor of 1. This flag will result in reduced\n" +
+      "       durability. Use with care.\n";
 
     @Override
     protected void processOptions(LinkedList<String> args) throws IOException {
-      CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, "f", "p");
+      CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, "f", "p", "l");
       cf.parse(args);
       setOverwrite(cf.getOpt("f"));
       setPreserve(cf.getOpt("p"));
+      setLazyPersist(cf.getOpt("l"));
       getRemoteDestination(args);
       // should have a -r option
       setRecursive(true);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbaa7dc2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
index 652c928..8b32eb8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FileStatus;
  *   %u: User name of owner
  *   %y: UTC date as &quot;yyyy-MM-dd HH:mm:ss&quot;
  *   %Y: Milliseconds since January 1, 1970 UTC
+ *   %l: Whether lazyPersist flag is set on the file.
  */
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
@@ -53,7 +54,8 @@ class Stat extends FsCommand {
   public static final String DESCRIPTION =
     "Print statistics about the file/directory at <path> " +
     "in the specified format. Format accepts filesize in blocks (%b), group name of owner(%g),
" +
-    "filename (%n), block size (%o), replication (%r), user name of owner(%u), modification
date (%y, %Y)\n";
+    "filename (%n), block size (%o), replication (%r), user name of owner(%u), modification
date (%y, %Y), " +
+    "lazyPersist flag (%l)\n";
 
   protected static final SimpleDateFormat timeFmt;
   static {
@@ -115,6 +117,9 @@ class Stat extends FsCommand {
           case 'Y':
             buf.append(stat.getModificationTime());
             break;
+          case 'l':
+            buf.append(stat.isLazyPersist());
+            break;
           default:
             // this leaves %<unknown> alone, which causes the potential for
             // future format options to break strings; should use %% to

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbaa7dc2/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
index 804b504..29d4174 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
@@ -436,7 +436,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-put \[-f\] \[-p\] &lt;localsrc&gt; \.\.\. &lt;dst&gt;
:\s*</expected-output>
+          <expected-output>^-put \[-f\] \[-p\] \[-l\] &lt;localsrc&gt; \.\.\.
&lt;dst&gt; :( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -444,15 +444,31 @@
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^\s*exists, unless the -f flag is given.( )*Passing -p preserves
access and( )*</expected-output>
+          <expected-output>^\s*exists, unless the -f flag is given.( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^\s*modification times, ownership and the mode. Passing
-f overwrites the( )*</expected-output>
+          <expected-output>^\s*Flags:( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*destination if it already exists.( )*</expected-output>
+          <expected-output>^\s*-p  Preserves access and modification times, ownership
and the mode.( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*-f  Overwrites the destination if it already exists.(
)*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*-l  Allow DataNode to lazily persist the file to disk.
Forces( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*replication factor of 1. This flag will result in reduced(
)*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*durability. Use with care.( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -467,7 +483,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-copyFromLocal \[-f\] \[-p\] &lt;localsrc&gt; \.\.\.
&lt;dst&gt; :\s*</expected-output>
+          <expected-output>^-copyFromLocal \[-f\] \[-p\] \[-l\] &lt;localsrc&gt;
\.\.\. &lt;dst&gt; :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -782,7 +798,7 @@
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*\(%y, %Y\)( )*</expected-output>
+          <expected-output>^( |\t)*\(%y, %Y\), lazyPersist flag \(\%l\)( )*</expected-output>
         </comparator>
       </comparators>
     </test>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbaa7dc2/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
index aac215c..b0fb070 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
@@ -21,4 +21,6 @@
     HDFS-6929. NN periodically unlinks lazy persist files with missing
     replicas from namespace. (Arpit Agarwal)
 
+    HDFS-6928. 'hdfs put' command should accept lazyPersist flag for testing.
+    (Arpit Agarwal)
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbaa7dc2/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
index 087c3ab..b8fba03 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
@@ -4554,6 +4554,42 @@
       </comparators>
     </test>
 
+    <test> <!-- TESTED -->
+      <description>put: The LazyPersist flag is set when requested</description>
+      <test-commands>
+        <command>-fs NAMENODE -mkdir /dirLp</command>
+        <command>-fs NAMENODE -put -l CLITEST_DATA/data15bytes /dirLp/data15bytes</command>
+        <command>-fs NAMENODE -stat %l /dirLp/data15bytes</command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm -r /dirLp/</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^true</expected-output>
+        </comparator>
+      </comparators>
+    </test>
+
+    <test> <!-- TESTED -->
+      <description>put: The LazyPersist flag is not set by default</description>
+      <test-commands>
+        <command>-fs NAMENODE -mkdir /dirLp</command>
+        <command>-fs NAMENODE -put CLITEST_DATA/data15bytes /dirLp/data15bytes</command>
+        <command>-fs NAMENODE -stat %l /dirLp/data15bytes</command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm -r /dirLp/</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^false</expected-output>
+        </comparator>
+      </comparators>
+    </test>
+
     <!-- Tests for copyFromLocal -->
     <test> <!-- TESTED -->
       <description>copyFromLocal: copying file into a file (absolute path)</description>


Mime
View raw message