hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhr...@apache.org
Subject svn commit: r633779 - in /hadoop/core/trunk: CHANGES.txt src/java/org/apache/hadoop/dfs/DataNode.java src/java/org/apache/hadoop/dfs/FSDataset.java
Date Wed, 05 Mar 2008 06:49:37 GMT
Author: dhruba
Date: Tue Mar  4 22:49:35 2008
New Revision: 633779

URL: http://svn.apache.org/viewvc?rev=633779&view=rev
Log:
HADOOP-2883. Write failures and data corruptions on HDFS files.
The write timeout is back to what it was on 0.15 release. Also, the
datnodes flushes the block file buffered output stream before
sending a positive ack for the packet back to the client. (dhruba)


Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java
    hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSDataset.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=633779&r1=633778&r2=633779&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue Mar  4 22:49:35 2008
@@ -242,6 +242,11 @@
     HADOOP-2931. IOException thrown by DFSOutputStream had wrong stack
     trace in some cases. (Michael Bieniosek via rangadi)
 
+    HADOOP-2883. Write failures and data corruptions on HDFS files.
+    The write timeout is back to what it was on 0.15 release. Also, the
+    datnodes flushes the block file buffered output stream before
+    sending a positive ack for the packet back to the client. (dhruba)
+
 Release 0.16.0 - 2008-02-07
 
   INCOMPATIBLE CHANGES

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java?rev=633779&r1=633778&r2=633779&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java Tue Mar  4 22:49:35 2008
@@ -1097,7 +1097,7 @@
           mirrorTarget = NetUtils.createSocketAddr(mirrorNode);
           mirrorSock = new Socket();
           try {
-            int timeoutValue = 3000 * numTargets + socketTimeout;
+            int timeoutValue = numTargets * socketTimeout;
             mirrorSock.connect(mirrorTarget, timeoutValue);
             mirrorSock.setSoTimeout(timeoutValue);
             mirrorSock.setSendBufferSize(DEFAULT_DATA_SOCKET_SIZE);
@@ -1898,6 +1898,11 @@
                          " from " + receiver.inAddr);
               }
               lastPacket = true;
+            } else {
+              // flush packet to disk before sending ack
+              if (!receiver.finalized) {
+                receiver.flush();
+              }
             }
 
             replyOut.writeLong(expected);
@@ -1982,6 +1987,12 @@
                        " of size " + block.getNumBytes() + 
                        " from " + receiver.inAddr);
             }
+            else if (!lastPacketInBlock) {
+              // flush packet to disk before sending ack
+              if (!receiver.finalized) {
+                receiver.flush();
+              }
+            }
 
             // send my status back to upstream datanode
             replyOut.writeLong(expected); // send seqno upstream
@@ -2159,6 +2170,16 @@
       }
     }
 
+    // flush block data and metadata files to disk.
+    void flush() throws IOException {
+      if (checksumOut != null) {
+        checksumOut.flush();
+      }
+      if (out != null) {
+        out.flush();
+      }
+    }
+
     /* receive a chunk: write it to disk & mirror it to another stream */
     private void receiveChunk( int len, byte[] checksumBuf, int checksumOff ) 
                               throws IOException {
@@ -2481,7 +2502,7 @@
       if (checksumOut != null) {
         checksumOut.flush();
       }
-      LOG.info("Changing block file offset from " + 
+      LOG.info("Changing block file offset of block " + block + " from " + 
                data.getChannelPosition(block, streams) +
                " to " + offsetInBlock +
                " meta file offset to " + offsetInChecksum);

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSDataset.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSDataset.java?rev=633779&r1=633778&r2=633779&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSDataset.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSDataset.java Tue Mar  4 22:49:35 2008
@@ -331,6 +331,14 @@
     }
 
     /**
+     * Returns the name of the temporary file for this block.
+     */
+    File getTmpFile(Block b) throws IOException {
+      File f = new File(tmpDir, b.getBlockName());
+      return f;
+    }
+
+    /**
      * Files used for copy-on-write. They need recovery when datanode
      * restarts.
      */
@@ -747,6 +755,18 @@
   public void setChannelPosition(Block b, BlockWriteStreams streams, 
                                  long dataOffset, long ckOffset) 
                                  throws IOException {
+    long size = 0;
+    synchronized (this) {
+      FSVolume vol = volumeMap.get(b).getVolume();
+      size = vol.getTmpFile(b).length();
+    }
+    if (size < dataOffset) {
+      String msg = "Trying to change block file offset of block " + b +
+                     " to " + dataOffset +
+                     " but actual size of file is " +
+                     size;
+      throw new IOException(msg);
+    }
     FileOutputStream file = (FileOutputStream) streams.dataOut;
     file.getChannel().position(dataOffset);
     file = (FileOutputStream) streams.checksumOut;
@@ -762,7 +782,7 @@
     }
     return vol.createTmpFile(blk);
   }
-  
+
   //
   // REMIND - mjc - eventually we should have a timeout system
   // in place to clean up block files left by abandoned clients.
@@ -898,7 +918,7 @@
   }
 
   /**
-   * check if a data diretory is healthy
+   * check if a data directory is healthy
    * @throws DiskErrorException
    */
   public void checkDataDir() throws DiskErrorException {



Mime
View raw message