hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r662537 - in /hadoop/core/trunk: ./ src/java/org/apache/hadoop/dfs/ src/java/org/apache/hadoop/fs/ src/java/org/apache/hadoop/fs/ftp/ src/java/org/apache/hadoop/fs/kfs/ src/java/org/apache/hadoop/fs/s3/ src/test/org/apache/hadoop/fs/
Date Mon, 02 Jun 2008 19:52:39 GMT
Author: cdouglas
Date: Mon Jun  2 12:52:38 2008
New Revision: 662537

URL: http://svn.apache.org/viewvc?rev=662537&view=rev
Log:
HADOOP-3250. Extend FileSystem API to allow appending to files.
Contributed by Tsz Wo (Nicholas), SZE.


Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
    hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java
    hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
    hadoop/core/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java
    hadoop/core/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java
    hadoop/core/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java
    hadoop/core/trunk/src/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
    hadoop/core/trunk/src/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java
    hadoop/core/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java
    hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystem.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=662537&r1=662536&r2=662537&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Jun  2 12:52:38 2008
@@ -114,6 +114,9 @@
 
     HADOOP-3246. Add FTPFileSystem.  (Ankur Goel via cutting)
 
+    HADOOP-3250. Extend FileSystem API to allow appending to files.
+    (Tsz Wo (Nicholas), SZE via cdouglas)
+
   IMPROVEMENTS
    
     HADOOP-2928. Remove deprecated FileSystem.getContentLength().

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java?rev=662537&r1=662536&r2=662537&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java Mon Jun  2
12:52:38 2008
@@ -128,6 +128,12 @@
           dfs.open(getPathName(f), bufferSize, verifyChecksum, statistics));
   }
 
+  /** This optional operation is not yet supported. */
+  public FSDataOutputStream append(Path f, int bufferSize,
+      Progressable progress) throws IOException {
+    throw new IOException("Not supported");
+  }
+
   public FSDataOutputStream create(Path f, FsPermission permission,
     boolean overwrite,
     int bufferSize, short replication, long blockSize,

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java?rev=662537&r1=662536&r2=662537&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java Mon Jun  2 12:52:38
2008
@@ -228,6 +228,12 @@
   @Override
   public void setWorkingDirectory(Path f) { }
 
+  /** This optional operation is not yet supported. */
+  public FSDataOutputStream append(Path f, int bufferSize,
+      Progressable progress) throws IOException {
+    throw new IOException("Not supported");
+  }
+
   @Override
   public FSDataOutputStream create(Path f, FsPermission permission,
                                    boolean overwrite, int bufferSize,

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java?rev=662537&r1=662536&r2=662537&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java Mon Jun  2 12:52:38 2008
@@ -522,6 +522,36 @@
   }
 
   /**
+   * Append to an existing file (optional operation).
+   * Same as append(f, getConf().getInt("io.file.buffer.size", 4096), null)
+   * @param f the existing file to be appended.
+   * @throws IOException
+   */
+  public FSDataOutputStream append(Path f) throws IOException {
+    return append(f, getConf().getInt("io.file.buffer.size", 4096), null);
+  }
+  /**
+   * Append to an existing file (optional operation).
+   * Same as append(f, bufferSize, null).
+   * @param f the existing file to be appended.
+   * @param bufferSize the size of the buffer to be used.
+   * @throws IOException
+   */
+  public FSDataOutputStream append(Path f, int bufferSize) throws IOException {
+    return append(f, bufferSize, null);
+  }
+
+  /**
+   * Append to an existing file (optional operation).
+   * @param f the existing file to be appended.
+   * @param bufferSize the size of the buffer to be used.
+   * @param progress for reporting progress if it is not null.
+   * @throws IOException
+   */
+  public abstract FSDataOutputStream append(Path f, int bufferSize,
+      Progressable progress) throws IOException;
+  
+  /**
    * Get replication.
    * 
    * @deprecated Use getFileStatus() instead

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java?rev=662537&r1=662536&r2=662537&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java Mon Jun  2 12:52:38
2008
@@ -104,7 +104,13 @@
   public FSDataInputStream open(Path f, int bufferSize) throws IOException {
     return fs.open(f, bufferSize);
   }
-  
+
+  /** {@inheritDoc} */
+  public FSDataOutputStream append(Path f, int bufferSize,
+      Progressable progress) throws IOException {
+    return fs.append(f, bufferSize, progress);
+  }
+
   /** {@inheritDoc} */
   @Override
   public FSDataOutputStream create(Path f, FsPermission permission,

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java?rev=662537&r1=662536&r2=662537&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java Mon Jun  2 12:52:38
2008
@@ -199,6 +199,12 @@
       }
     }
   
+    /** This optional operation is not yet supported. */
+    public FSDataOutputStream append(Path f, int bufferSize,
+        Progressable progress) throws IOException {
+      throw new IOException("Not supported");
+    }
+
     /**
      * @param permission Currently ignored.
      */

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=662537&r1=662536&r2=662537&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java Mon Jun  2 12:52:38
2008
@@ -183,8 +183,8 @@
   class LocalFSFileOutputStream extends OutputStream {
     FileOutputStream fos;
     
-    public LocalFSFileOutputStream(Path f) throws IOException {
-      this.fos = new FileOutputStream(pathToFile(f));
+    private LocalFSFileOutputStream(Path f, boolean append) throws IOException {
+      this.fos = new FileOutputStream(pathToFile(f), append);
     }
     
     /*
@@ -209,6 +209,20 @@
     }
   }
   
+  /** {@inheritDoc} */
+  public FSDataOutputStream append(Path f, int bufferSize,
+      Progressable progress) throws IOException {
+    if (!exists(f)) {
+      throw new FileNotFoundException("File " + f + " not found.");
+    }
+    if (getFileStatus(f).isDir()) {
+      throw new IOException("Cannot append to a diretory (=" + f + " ).");
+    }
+    return new FSDataOutputStream(new BufferedOutputStream(
+        new LocalFSFileOutputStream(f, true), bufferSize), statistics);
+  }
+
+  /** {@inheritDoc} */
   public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize,
                                    short replication, long blockSize, Progressable progress)
     throws IOException {
@@ -219,9 +233,8 @@
     if (parent != null && !mkdirs(parent)) {
       throw new IOException("Mkdirs failed to create " + parent.toString());
     }
-    return new FSDataOutputStream(
-        new BufferedOutputStream(new LocalFSFileOutputStream(f), bufferSize),
-        statistics);
+    return new FSDataOutputStream(new BufferedOutputStream(
+        new LocalFSFileOutputStream(f, false), bufferSize), statistics);
   }
 
   /** {@inheritDoc} */

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java?rev=662537&r1=662536&r2=662537&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java Mon Jun  2 12:52:38
2008
@@ -243,6 +243,12 @@
     return fos;
   }
 
+  /** This optional operation is not yet supported. */
+  public FSDataOutputStream append(Path f, int bufferSize,
+      Progressable progress) throws IOException {
+    throw new IOException("Not supported");
+  }
+  
   /**
    * Convenience method, so that we don't open a new connection when using this
    * method from within another method. Otherwise every API invocation incurs

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java?rev=662537&r1=662536&r2=662537&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java Mon Jun  2 12:52:38
2008
@@ -183,6 +183,12 @@
         }
     }
     
+    /** This optional operation is not yet supported. */
+    public FSDataOutputStream append(Path f, int bufferSize,
+        Progressable progress) throws IOException {
+      throw new IOException("Not supported");
+    }
+
     public FSDataOutputStream create(Path file, FsPermission permission,
                                      boolean overwrite, int bufferSize,
 				     short replication, long blockSize, Progressable progress)

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java?rev=662537&r1=662536&r2=662537&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java Mon Jun  2 12:52:38
2008
@@ -179,6 +179,12 @@
     return ret.toArray(new FileStatus[0]);
   }
 
+  /** This optional operation is not yet supported. */
+  public FSDataOutputStream append(Path f, int bufferSize,
+      Progressable progress) throws IOException {
+    throw new IOException("Not supported");
+  }
+
   /**
    * @param permission Currently ignored.
    */

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystem.java?rev=662537&r1=662536&r2=662537&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystem.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystem.java Mon Jun  2 12:52:38
2008
@@ -29,12 +29,26 @@
     = System.getProperty("test.build.data","build/test/data/work-dir/localfs");
 
 
-  private void writeFile(FileSystem fs, Path name) throws IOException {
+  static void writeFile(FileSystem fs, Path name) throws IOException {
     FSDataOutputStream stm = fs.create(name);
     stm.writeBytes("42\n");
     stm.close();
   }
   
+  static String readFile(FileSystem fs, Path name) throws IOException {
+    byte[] b = new byte[1024];
+    int offset = 0;
+    FSDataInputStream in = fs.open(name);
+    for(int remaining, n;
+        (remaining = b.length - offset) > 0 && (n = in.read(b, offset, remaining))
!= -1;
+        offset += n); 
+    in.close();
+
+    String s = new String(b, 0, offset);
+    System.out.println("s=" + s);
+    return s;
+  }
+
   private void cleanupFile(FileSystem fs, Path name) throws IOException {
     assertTrue(fs.exists(name));
     fs.delete(name, true);
@@ -140,4 +154,54 @@
     cleanupFile(fs, path);
   }
 
+  public void testAppend() throws IOException {
+    Configuration conf = new Configuration();
+    final String dir = TEST_ROOT_DIR + "/append";
+    LocalFileSystem fs = FileSystem.getLocal(conf);
+
+    //normal case
+    {
+      Path f = new Path(dir, "f");
+      FSDataOutputStream out = fs.create(f);
+      out.writeBytes("something");
+      out.close();
+      assertEquals("something", readFile(fs, f));
+  
+      out = fs.append(f);
+      out.writeBytes(" more");
+      out.close();
+      assertEquals("something more", readFile(fs, f));
+  
+      out = fs.append(f);
+      out.writeBytes(" and more");
+      out.close();
+      assertEquals("something more and more", readFile(fs, f));
+
+      cleanupFile(fs, f);
+    }
+
+    //file not found case
+    {
+      Path f = new Path(dir, "fileNotFound");
+      try {
+        fs.append(f);
+        assertTrue(false);
+      }
+      catch(FileNotFoundException e) {
+        System.out.println("good: " + e);
+      }
+    }
+
+    //append to dir case
+    {
+      Path f = new Path(dir);
+      try {
+        fs.append(f);
+        assertTrue(false);
+      }
+      catch(IOException e) {
+        System.out.println("good: " + e);
+      }
+    }
+  }
 }



Mime
View raw message