Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Reply-To: common-dev@hadoop.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1211578 - in /hadoop/common/branches/branch-1: ./
 src/hdfs/org/apache/hadoop/hdfs/ src/hdfs/org/apache/hadoop/hdfs/protocol/
 src/hdfs/org/apache/hadoop/hdfs/server/datanode/
 src/hdfs/org/apache/hadoop/hdfs/server/namenode/
Date: Wed, 07 Dec 2011 18:08:46 -0000
To: common-commits@hadoop.apache.org
From: eli@apache.org
Message-Id: <20111207180847.3B5F52388860@eris.apache.org>

Author: eli
Date: Wed Dec  7 18:08:46 2011
New Revision: 1211578

URL: http://svn.apache.org/viewvc?rev=1211578&view=rev
Log:
HDFS-2638. Improve a block recovery log. Contributed by Eli Collins

Modified:
    hadoop/common/branches/branch-1/CHANGES.txt
    hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java
    hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/protocol/Block.java
    hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java
    hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java

Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1211578&r1=1211577&r2=1211578&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Wed Dec  7 18:08:46 2011
@@ -105,6 +105,8 @@ Release 1.1.0 - unreleased
 
     MAPREDUCE-2103. task-controller shouldn't require o-r permissions. (eli)
 
+    HDFS-2638. Improve a block recovery log. (eli)
+
 Release 1.0.0 - unreleased
 
   NEW FEATURES

Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java?rev=1211578&r1=1211577&r2=1211578&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java (original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java Wed Dec  7 18:08:46 2011
@@ -2823,7 +2823,7 @@ public class DFSClient implements FSCons
               // get new block from namenode.
               if (blockStream == null) {
                 LOG.debug("Allocating new block");
-                nodes = nextBlockOutputStream(src); 
+                nodes = nextBlockOutputStream();
                 this.setName("DataStreamer for file " + src +
                              " block " + block);
                 response = new ResponseProcessor(nodes);
@@ -3354,7 +3354,7 @@ public class DFSClient implements FSCons
      * Must get block ID and the IDs of the destinations from the namenode.
      * Returns the list of target datanodes.
      */
-    private DatanodeInfo[] nextBlockOutputStream(String client) throws IOException {
+    private DatanodeInfo[] nextBlockOutputStream() throws IOException {
       LocatedBlock lb = null;
       boolean retry = false;
       DatanodeInfo[] nodes;

Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/protocol/Block.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/protocol/Block.java?rev=1211578&r1=1211577&r2=1211578&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/protocol/Block.java (original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/protocol/Block.java Wed Dec  7 18:08:46 2011
@@ -116,6 +116,10 @@ public class Block implements Writable, 
     generationStamp = stamp;
   }
 
+  public Block getWithWildcardGS() {
+    return new Block(blockId, numBytes, GenerationStamp.WILDCARD_STAMP);
+  }
+
   /**
    */
   public String toString() {

Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1211578&r1=1211577&r2=1211578&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java Wed Dec  7 18:08:46 2011
@@ -1875,9 +1875,7 @@ public class DataNode extends Configured
     // This can happen if the namenode and client start recovering the same
     // file at the same time.
     synchronized (ongoingRecovery) {
-      Block tmp = new Block();
-      tmp.set(block.getBlockId(), block.getNumBytes(), GenerationStamp.WILDCARD_STAMP);
-      if (ongoingRecovery.get(tmp) != null) {
+      if (ongoingRecovery.get(block.getWithWildcardGS()) != null) {
         String msg = "Block " + block + " is already being recovered, " +
                      " ignoring this request to recover it.";
         LOG.info(msg);

Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1211578&r1=1211577&r2=1211578&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Wed Dec  7 18:08:46 2011
@@ -2252,8 +2252,7 @@ public class FSNamesystem implements FSC
 
     if (deleteblock) {
       pendingFile.removeBlock(lastblock);
-    }
-    else {
+    } else {
       // update last block, construct newblockinfo and add it to the blocks map
       lastblock.set(lastblock.getBlockId(), newlength, newgenerationstamp);
       final BlockInfo newblockinfo = blocksMap.addINode(lastblock, pendingFile);
@@ -5381,7 +5380,13 @@ public class FSNamesystem implements FSC
     }
     BlockInfo storedBlock = blocksMap.getStoredBlock(block);
     if (storedBlock == null) {
-      String msg = block + " is already commited, storedBlock == null.";
+      BlockInfo match =
+        blocksMap.getStoredBlock(block.getWithWildcardGS());
+      String msg = (match == null)
+        ? block + " is missing"
+        : block + " has out of date GS " + block.getGenerationStamp() +
+                  " found " + match.getGenerationStamp() +
+                  ", may already be committed";
       LOG.info(msg);
       throw new IOException(msg);
     }