hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhr...@apache.org
Subject svn commit: r663326 - in /hadoop/core/trunk: ./ src/java/org/apache/hadoop/dfs/ src/java/org/apache/hadoop/fs/ src/test/org/apache/hadoop/dfs/
Date Wed, 04 Jun 2008 17:45:51 GMT
Author: dhruba
Date: Wed Jun  4 10:45:50 2008
New Revision: 663326

URL: http://svn.apache.org/viewvc?rev=663326&view=rev
Log:
HADOOP-3177. Implement Syncable interface for FileSystem.
(Tsz Wo (Nicholas), SZE via dhruba)


Added:
    hadoop/core/trunk/src/java/org/apache/hadoop/fs/Syncable.java
Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java
    hadoop/core/trunk/src/java/org/apache/hadoop/fs/FSDataOutputStream.java
    hadoop/core/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java
    hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestAbandonBlock.java
    hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFileAppend.java
    hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFileCreation.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=663326&r1=663325&r2=663326&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed Jun  4 10:45:50 2008
@@ -126,6 +126,9 @@
     HADOOP-3250. Extend FileSystem API to allow appending to files.
     (Tsz Wo (Nicholas), SZE via cdouglas)
 
+    HADOOP-3177. Implement Syncable interface for FileSystem.
+    (Tsz Wo (Nicholas), SZE via dhruba)
+
   IMPROVEMENTS
    
     HADOOP-2928. Remove deprecated FileSystem.getContentLength().

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java?rev=663326&r1=663325&r2=663326&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java Wed Jun  4 10:45:50 2008
@@ -1699,7 +1699,7 @@
    * datanode from the original pipeline. The DataStreamer now
    * starts sending packets from the dataQueue.
   ****************************************************************/
-  class DFSOutputStream extends FSOutputSummer {
+  class DFSOutputStream extends FSOutputSummer implements Syncable {
     private Socket s;
     boolean closed = false;
   
@@ -2497,7 +2497,7 @@
      * that data has been flushed to persistent store on the 
      * datanode. Block allocations are persisted on namenode.
      */
-    public synchronized void fsync() throws IOException {
+    public synchronized void sync() throws IOException {
       try {
         /* Record current blockOffset. This might be changed inside
          * flushBuffer() where a partial checksum chunk might be flushed.

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/FSDataOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/FSDataOutputStream.java?rev=663326&r1=663325&r2=663326&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/FSDataOutputStream.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/FSDataOutputStream.java Wed Jun  4 10:45:50
2008
@@ -22,7 +22,7 @@
 /** Utility that wraps a {@link OutputStream} in a {@link DataOutputStream},
  * buffers output through a {@link BufferedOutputStream} and creates a checksum
  * file. */
-public class FSDataOutputStream extends DataOutputStream {
+public class FSDataOutputStream extends DataOutputStream implements Syncable {
   private OutputStream wrappedStream;
 
   private static class PositionCache extends FilterOutputStream {
@@ -83,4 +83,11 @@
   public OutputStream getWrappedStream() {
     return wrappedStream;
   }
+
+  /** {@inheritDoc} */
+  public void sync() throws IOException {
+    if (wrappedStream instanceof Syncable) {
+      ((Syncable)wrappedStream).sync();
+    }
+  }
 }

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=663326&r1=663325&r2=663326&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java Wed Jun  4 10:45:50
2008
@@ -180,7 +180,7 @@
   /*********************************************************
    * For create()'s FSOutputStream.
    *********************************************************/
-  class LocalFSFileOutputStream extends OutputStream {
+  class LocalFSFileOutputStream extends OutputStream implements Syncable {
     FileOutputStream fos;
     
     private LocalFSFileOutputStream(Path f, boolean append) throws IOException {
@@ -207,6 +207,11 @@
         throw new FSError(e);                // assume native fs error
       }
     }
+
+    /** {@inheritDoc} */
+    public void sync() throws IOException {
+      fos.getFD().sync();      
+    }
   }
   
   /** {@inheritDoc} */

Added: hadoop/core/trunk/src/java/org/apache/hadoop/fs/Syncable.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/Syncable.java?rev=663326&view=auto
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/Syncable.java (added)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/Syncable.java Wed Jun  4 10:45:50 2008
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+
+/** This interface declare the sync() operation. */
+public interface Syncable {
+  /**
+   * Synchronize all buffer with the underlying devices.
+   * @throws IOException
+   */
+  public void sync() throws IOException;
+}
\ No newline at end of file

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestAbandonBlock.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestAbandonBlock.java?rev=663326&r1=663325&r2=663326&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestAbandonBlock.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestAbandonBlock.java Wed Jun  4 10:45:50
2008
@@ -24,7 +24,6 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.*;
 import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.net.NetUtils;
 
 public class TestAbandonBlock extends junit.framework.TestCase {
   public static final Log LOG = LogFactory.getLog(TestAbandonBlock.class);
@@ -33,12 +32,6 @@
   static final String FILE_NAME_PREFIX
       = "/" + TestAbandonBlock.class.getSimpleName() + "_"; 
 
-  private void flushFile(FSDataOutputStream stm) throws IOException {
-    DFSClient.DFSOutputStream dfstream = (DFSClient.DFSOutputStream)
-                                            (stm.getWrappedStream());
-    dfstream.fsync();
-  }
-
   public void testAbandonBlock() throws IOException {
     MiniDFSCluster cluster = new MiniDFSCluster(CONF, 2, true, null);
     FileSystem fs = cluster.getFileSystem();
@@ -51,7 +44,7 @@
       for(int i = 0; i < 1024; i++) {
         fout.write(123);
       }
-      flushFile(fout);
+      fout.sync();
   
       //try reading the block by someone
       DFSClient dfsclient = new DFSClient(CONF);

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFileAppend.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFileAppend.java?rev=663326&r1=663325&r2=663326&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFileAppend.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFileAppend.java Wed Jun  4 10:45:50
2008
@@ -63,12 +63,6 @@
     return stm;
   }
 
-  private void flushFile(FSDataOutputStream stm) throws IOException {
-    DFSClient.DFSOutputStream dfstream = (DFSClient.DFSOutputStream)
-                                            (stm.getWrappedStream());
-    dfstream.fsync();
-  }
-
   //
   // writes to file but does not close it
   //
@@ -234,14 +228,14 @@
       // write to file
       int mid = fileSize/2;
       stm.write(fileContents, 0, mid);
-      flushFile(stm);
+      stm.sync();
       System.out.println("Wrote and Flushed first part of file.");
 
       // write the remainder of the file
       stm.write(fileContents, mid, fileSize - mid);
       System.out.println("Written second part of file");
-      flushFile(stm);
-      flushFile(stm); // two consecutive flushes is being tested here.
+      stm.sync();
+      stm.sync();
       System.out.println("Wrote and Flushed second part of file.");
 
       // verify that full blocks are sane
@@ -287,7 +281,7 @@
       int start = 0;
       for (start = 0; (start + 29) < fileSize; ) {
         stm.write(fileContents, start, 29);
-        flushFile(stm);
+        stm.sync();
         start += 29;
       }
       stm.write(fileContents, start, fileSize-start);

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFileCreation.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFileCreation.java?rev=663326&r1=663325&r2=663326&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFileCreation.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFileCreation.java Wed Jun  4 10:45:50
2008
@@ -62,12 +62,6 @@
     return stm;
   }
 
-  private void flushFile(FSDataOutputStream stm) throws IOException {
-    DFSClient.DFSOutputStream dfstream = (DFSClient.DFSOutputStream)
-                                            (stm.getWrappedStream());
-    dfstream.fsync();
-  }
-
   //
   // writes to file but does not close it
   //
@@ -439,7 +433,7 @@
 
       // write two full blocks.
       writeFile(stm, numBlocks * blockSize);
-      flushFile(stm);
+      stm.sync();
 
       // rename file wile keeping it open.
       Path fileRenamed = new Path("/filestatusRenamed.dat");
@@ -657,7 +651,7 @@
       final Path fpath = new Path(f);
       FSDataOutputStream out = TestFileCreation.createFile(dfs, fpath, DATANODE_NUM);
       out.write("something".getBytes());
-      ((DFSClient.DFSOutputStream)out.getWrappedStream()).fsync();
+      out.sync();
 
       // set the soft and hard limit to be 1 second so that the
       // namenode triggers lease recovery



Mime
View raw message