hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hair...@apache.org
Subject svn commit: r902330 - in /hadoop/hdfs/trunk: CHANGES.txt src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Date Sat, 23 Jan 2010 01:35:02 GMT
Author: hairong
Date: Sat Jan 23 01:35:01 2010
New Revision: 902330

URL: http://svn.apache.org/viewvc?rev=902330&view=rev
Log:
HDFS-800. The last block of a file under construction may change to the COMPLETE state in
response to getAdditionalBlock or completeFileInternal. Contributed by Hairong Kuang.

Modified:
    hadoop/hdfs/trunk/CHANGES.txt
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java

Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=902330&r1=902329&r2=902330&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Sat Jan 23 01:35:01 2010
@@ -38,6 +38,10 @@
 
     HDFS-902 Move contrib/raid to MapReduce. (Eli Collins via omalley)
 
+    HDFS-800. The last block of a file under construction may change to the
+    COMPLETE state in response to getAdditionalBlock or completeFileInternal.
+    (hairong)
+
   OPTIMIZATIONS
 
   BUG FIXES

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java?rev=902330&r1=902329&r2=902330&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java Sat
Jan 23 01:35:01 2010
@@ -245,26 +245,23 @@
   }
 
   /**
-   * Commit the last block of the file.
+   * Commit a block of a file
    * 
    * @param fileINode file inode
+   * @param block block to be committed
    * @param commitBlock - contains client reported block length and generation
    * @throws IOException if the block does not have at least a minimal number
    * of replicas reported from data-nodes.
    */
-  void commitLastBlock(INodeFileUnderConstruction fileINode, 
+  private void commitBlock(INodeFileUnderConstruction fileINode,
+                       BlockInfoUnderConstruction block,
                        Block commitBlock) throws IOException {
-    if(commitBlock == null)
-      return; // not committing, this is a block allocation retry
-    BlockInfo lastBlock = fileINode.getLastBlock();
-    if(lastBlock == null)
-      return; // no blocks in file yet
-    if(lastBlock.isComplete())
-      return; // already completed (e.g. by syncBlock)
-    assert lastBlock.getNumBytes() <= commitBlock.getNumBytes() :
+    if (block.getBlockUCState() == BlockUCState.COMMITTED)
+      return;
+    assert block.getNumBytes() <= commitBlock.getNumBytes() :
       "commitBlock length is less than the stored one "
-      + commitBlock.getNumBytes() + " vs. " + lastBlock.getNumBytes();
-    ((BlockInfoUnderConstruction)lastBlock).commitBlock(commitBlock);
+      + commitBlock.getNumBytes() + " vs. " + block.getNumBytes();
+    block.commitBlock(commitBlock);
     
     // Adjust disk space consumption if required
     long diff = fileINode.getPreferredBlockSize() - commitBlock.getNumBytes();    
@@ -281,6 +278,32 @@
       }
     }
   }
+  
+  /**
+   * Commit the last block of the file and mark it as complete if it has
+   * meets the minimum replication requirement
+   * 
+   * @param fileINode file inode
+   * @param commitBlock - contains client reported block length and generation
+   * @throws IOException if the block does not have at least a minimal number
+   * of replicas reported from data-nodes.
+   */
+  void commitOrCompleteLastBlock(INodeFileUnderConstruction fileINode, 
+      Block commitBlock) throws IOException {
+    
+    if(commitBlock == null)
+      return; // not committing, this is a block allocation retry
+    BlockInfo lastBlock = fileINode.getLastBlock();
+    if(lastBlock == null)
+      return; // no blocks in file yet
+    if(lastBlock.isComplete())
+      return; // already completed (e.g. by syncBlock)
+    
+    commitBlock(fileINode, (BlockInfoUnderConstruction)lastBlock, commitBlock);
+
+    if(countNodes(lastBlock).liveReplicas() >= minReplication)
+      completeBlock(fileINode,fileINode.numBlocks()-1);
+  }
 
   /**
    * Convert a specified block of the file to a complete block.

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=902330&r1=902329&r2=902330&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Sat
Jan 23 01:35:01 2010
@@ -1326,8 +1326,8 @@
 
       INodeFileUnderConstruction pendingFile  = checkLease(src, clientName);
 
-      // commit the last block
-      blockManager.commitLastBlock(pendingFile, previous);
+      // commit the last block and complete it if it has minimum replicas
+      blockManager.commitOrCompleteLastBlock(pendingFile, previous);
 
       //
       // If we fail this, bad things happen!
@@ -1362,9 +1362,6 @@
         throw new NotReplicatedYetException("Not replicated yet:" + src);
       }
 
-      // complete the penultimate block
-      blockManager.completeBlock(pendingFile, pendingFile.numBlocks()-2);
-
       // allocate new block record block locations in INode.
       newBlock = allocateBlock(src, pathINodes, targets);
       
@@ -1480,8 +1477,8 @@
       return CompleteFileStatus.OPERATION_FAILED;
     } 
 
-    // commit the last block
-    blockManager.commitLastBlock(pendingFile, last);
+    // commit the last block and complete it if it has minimum replicas
+    blockManager.commitOrCompleteLastBlock(pendingFile, last);
 
     if (!checkFileProgress(pendingFile, true)) {
       return CompleteFileStatus.STILL_WAITING;
@@ -1541,7 +1538,7 @@
       // check all blocks of the file.
       //
       for (BlockInfo block: v.getBlocks()) {
-        if (!blockManager.checkMinReplication(block)) {
+        if (!block.isComplete()) {
           LOG.info("BLOCK* NameSystem.checkFileProgress: "
               + "block " + block + " has not reached minimal replication "
               + blockManager.minReplication);
@@ -1553,7 +1550,7 @@
       // check the penultimate block of this file
       //
       BlockInfo b = v.getPenultimateBlock();
-      if (b != null && !blockManager.checkMinReplication(b)) {
+      if (b != null && !b.isComplete()) {
         LOG.info("BLOCK* NameSystem.checkFileProgress: "
             + "block " + b + " has not reached minimal replication "
             + blockManager.minReplication);
@@ -2020,16 +2017,11 @@
       INodeFileUnderConstruction pendingFile) throws IOException {
     leaseManager.removeLease(pendingFile.getClientName(), src);
 
-    // complete the penultimate block
-    blockManager.completeBlock(pendingFile, pendingFile.numBlocks()-2);
-
     // The file is no longer pending.
     // Create permanent INode, update blocks
     INodeFile newFile = pendingFile.convertToInodeFile();
     dir.replaceNode(src, pendingFile, newFile);
 
-    // complete last block of the file
-    blockManager.completeBlock(newFile, newFile.numBlocks()-1);
     // close file and persist block allocations for this file
     dir.closeFile(src, newFile);
 
@@ -2111,8 +2103,8 @@
       return;
     }
 
-    // commit the last block
-    blockManager.commitLastBlock(pendingFile, storedBlock);
+    // commit the last block and complete it if it has minimum replicas
+    blockManager.commitOrCompleteLastBlock(pendingFile, storedBlock);
 
     //remove lease, close file
     finalizeINodeFileUnderConstruction(src, pendingFile);



Mime
View raw message