hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jiten...@apache.org
Subject svn commit: r1199035 - in /hadoop/common/branches/branch-0.20-security: CHANGES.txt src/core/org/apache/hadoop/fs/FileSystem.java src/core/org/apache/hadoop/io/SequenceFile.java src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
Date Tue, 08 Nov 2011 01:00:55 GMT
Author: jitendra
Date: Tue Nov  8 01:00:55 2011
New Revision: 1199035

URL: http://svn.apache.org/viewvc?rev=1199035&view=rev
Log:
HADOOP-6840. Support non-recursive create() in FileSystem & SequenceFile.Writer. Contributed
by Nicolas Spiegelberg.

Modified:
    hadoop/common/branches/branch-0.20-security/CHANGES.txt
    hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/fs/FileSystem.java
    hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/io/SequenceFile.java
    hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java

Modified: hadoop/common/branches/branch-0.20-security/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/CHANGES.txt?rev=1199035&r1=1199034&r2=1199035&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20-security/CHANGES.txt Tue Nov  8 01:00:55 2011
@@ -78,6 +78,9 @@ Release 0.20.205.1 - unreleased
     HDFS-617. Support for non-recursive create() in HDFS. 
     (Kan Zhang via jitendra)
 
+    HADOOP-6840. Support non-recursive create() in FileSystem & 
+    SequenceFile.Writer. (Nicolas Spiegelberg via jitendra)
+
   BUG FIXES
 
     HADOOP-7740. Fixed security audit logger configuration. 

Modified: hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/fs/FileSystem.java?rev=1199035&r1=1199034&r2=1199035&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/fs/FileSystem.java
(original)
+++ hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/fs/FileSystem.java
Tue Nov  8 01:00:55 2011
@@ -579,6 +579,54 @@ public abstract class FileSystem extends
       Progressable progress) throws IOException;
 
   /**
+  * Opens an FSDataOutputStream at the indicated Path with write-progress
+  * reporting. Same as create(), except fails if parent directory doesn't
+  * already exist.
+  * @param f the file name to open
+  * @param overwrite if a file with this name already exists, then if true,
+  * the file will be overwritten, and if false an error will be thrown.
+  * @param bufferSize the size of the buffer to be used.
+  * @param replication required block replication for the file.
+  * @param blockSize
+  * @param progress
+  * @throws IOException
+  * @see #setPermission(Path, FsPermission)
+  * @deprecated API only for 0.20-append
+  */
+  @Deprecated
+  public FSDataOutputStream createNonRecursive(Path f,
+      boolean overwrite,
+      int bufferSize, short replication, long blockSize,
+      Progressable progress) throws IOException {
+    return this.createNonRecursive(f, FsPermission.getDefault(),
+        overwrite, bufferSize, replication, blockSize, progress);
+  }
+  
+  /**
+  * Opens an FSDataOutputStream at the indicated Path with write-progress
+  * reporting. Same as create(), except fails if parent directory doesn't
+  * already exist.
+  * @param f the file name to open
+  * @param permission
+  * @param overwrite if a file with this name already exists, then if true,
+  * the file will be overwritten, and if false an error will be thrown.
+  * @param bufferSize the size of the buffer to be used.
+  * @param replication required block replication for the file.
+  * @param blockSize
+  * @param progress
+  * @throws IOException
+  * @see #setPermission(Path, FsPermission)
+  * @deprecated API only for 0.20-append
+  */
+  @Deprecated
+  public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
+      boolean overwrite,
+      int bufferSize, short replication, long blockSize,
+      Progressable progress) throws IOException {
+    throw new IOException("createNonRecursive unsupported for this filesystem");
+  }
+
+  /**
    * Creates the given Path as a brand-new zero-length file.  If
    * create fails, or if it already existed, return false.
    */

Modified: hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/io/SequenceFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/io/SequenceFile.java?rev=1199035&r1=1199034&r2=1199035&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/io/SequenceFile.java
(original)
+++ hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/io/SequenceFile.java
Tue Nov  8 01:00:55 2011
@@ -408,6 +408,55 @@ public class SequenceFile {
 
   /**
    * Construct the preferred type of SequenceFile Writer.
+   * @param fs The configured filesystem.
+   * @param conf The configuration.
+   * @param name The name of the file.
+   * @param keyClass The 'key' type.
+   * @param valClass The 'value' type.
+   * @param bufferSize buffer size for the underlaying outputstream.
+   * @param replication replication factor for the file.
+   * @param blockSize block size for the file.
+   * @param createParent create parent directory if non-existent
+   * @param compressionType The compression type.
+   * @param codec The compression codec.
+   * @param progress The Progressable object to track progress.
+   * @param metadata The metadata of the file.
+   * @return Returns the handle to the constructed SequenceFile Writer.
+   * @throws IOException
+   */
+  public static Writer
+    createWriter(FileSystem fs, Configuration conf, Path name,
+                 Class keyClass, Class valClass, int bufferSize,
+                 short replication, long blockSize, boolean createParent,
+                 CompressionType compressionType, CompressionCodec codec,
+                 Metadata metadata) throws IOException {
+    if ((codec instanceof GzipCodec) &&
+        !NativeCodeLoader.isNativeCodeLoaded() &&
+        !ZlibFactory.isNativeZlibLoaded(conf)) {
+      throw new IllegalArgumentException("SequenceFile doesn't work with " +
+                                         "GzipCodec without native-hadoop code!");
+    }
+
+    switch (compressionType) {
+    case NONE:
+      return new Writer(conf, 
+          fs.createNonRecursive(name, true, bufferSize, replication, blockSize, null),
+          keyClass, valClass, metadata).ownStream();
+    case RECORD:
+      return new RecordCompressWriter(conf, 
+          fs.createNonRecursive(name, true, bufferSize, replication, blockSize, null),
+          keyClass, valClass, codec, metadata).ownStream();
+    case BLOCK:
+      return new BlockCompressWriter(conf,
+          fs.createNonRecursive(name, true, bufferSize, replication, blockSize, null),
+          keyClass, valClass, codec, metadata).ownStream();
+    default:
+      return null;
+    }
+  } 
+  
+  /**
+   * Construct the preferred type of SequenceFile Writer.
    * @param fs The configured filesystem. 
    * @param conf The configuration.
    * @param name The name of the file. 
@@ -849,7 +898,7 @@ public class SequenceFile {
     }
 
     /** Write to an arbitrary stream using a specified buffer size. */
-    private Writer(Configuration conf, FSDataOutputStream out, 
+    Writer(Configuration conf, FSDataOutputStream out, 
                    Class keyClass, Class valClass, Metadata metadata)
       throws IOException {
       this.ownOutputStream = false;
@@ -876,6 +925,8 @@ public class SequenceFile {
     boolean isCompressed() { return compress; }
     boolean isBlockCompressed() { return false; }
     
+    Writer ownStream() { this.ownOutputStream = true; return this; }
+    
     /** Write and flush the file header. */
     void writeFileHeader() 
       throws IOException {
@@ -1096,7 +1147,7 @@ public class SequenceFile {
     }
     
     /** Write to an arbitrary stream using a specified buffer size. */
-    private RecordCompressWriter(Configuration conf, FSDataOutputStream out,
+    RecordCompressWriter(Configuration conf, FSDataOutputStream out,
                                  Class keyClass, Class valClass, CompressionCodec codec,
Metadata metadata)
       throws IOException {
       this.ownOutputStream = false;
@@ -1221,12 +1272,12 @@ public class SequenceFile {
     }
     
     /** Write to an arbitrary stream using a specified buffer size. */
-    private BlockCompressWriter(Configuration conf, FSDataOutputStream out,
+    BlockCompressWriter(Configuration conf, FSDataOutputStream out,
                                 Class keyClass, Class valClass, CompressionCodec codec, Metadata
metadata)
       throws IOException {
       this.ownOutputStream = false;
       super.init(null, conf, out, keyClass, valClass, true, codec, metadata);
-      init(1000000);
+      init(conf.getInt("io.seqfile.compress.blocksize", 1000000));
       
       initializeFileHeader();
       writeFileHeader();

Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java?rev=1199035&r1=1199034&r2=1199035&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
(original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
Tue Nov  8 01:00:55 2011
@@ -189,6 +189,7 @@ public class DistributedFileSystem exten
    * Same as create(), except fails if parent directory doesn't already exist.
    * @see #create(Path, FsPermission, boolean, int, short, long, Progressable)
    */
+  @Override
   public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
       boolean overwrite,
       int bufferSize, short replication, long blockSize, 



Mime
View raw message