hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhr...@apache.org
Subject svn commit: r577372 - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/dfs/BlocksMap.java src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java
Date Wed, 19 Sep 2007 16:58:49 GMT
Author: dhruba
Date: Wed Sep 19 09:58:49 2007
New Revision: 577372

URL: http://svn.apache.org/viewvc?rev=577372&view=rev
Log:
HADOOP-1904. The Namenode encounters an exception because the
list of blocks per datanode-descriptor was corrupted.


Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/BlocksMap.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=577372&r1=577371&r2=577372&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Wed Sep 19 09:58:49 2007
@@ -83,6 +83,10 @@
 
   BUG FIXES
 
+    HADOOP-1904. The Namenode encounters an exception because the
+    list of blocks per datanode-descriptor was corrupted.
+    (Konstantin Shvachko via dhruba)
+
     HADOOP-1762. The Namenode fsimage does not contain a list of
     Datanodes.  (Raghu Angadi via dhruba)
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/BlocksMap.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/BlocksMap.java?rev=577372&r1=577371&r2=577372&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/BlocksMap.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/BlocksMap.java Wed Sep 19 09:58:49
2007
@@ -142,14 +142,13 @@
      * Add data-node this block belongs to.
      */
     boolean addNode(DatanodeDescriptor node) {
-      int dnIndex = this.findDatanode(node);
-      if(dnIndex >= 0) // the node is already there
+      if(findDatanode(node) >= 0) // the node is already there
         return false;
       // find the last null node
       int lastNode = ensureCapacity(1);
       setDatanode(lastNode, node);
-      setNext(lastNode, null); 
-      setPrevious(lastNode, null); 
+      setNext(lastNode, null);
+      setPrevious(lastNode, null);
       return true;
     }
 
@@ -160,6 +159,8 @@
       int dnIndex = this.findDatanode(node);
       if(dnIndex < 0) // the node is not found
         return false;
+      assert getPrevious(dnIndex) == null && getNext(dnIndex) == null : 
+        "Block is still in the list and must be removed first.";
       // find the last not null node
       int lastNode = numNodes()-1; 
       // replace current node triplet by the lastNode one 
@@ -199,13 +200,12 @@
     BlockInfo listInsert(BlockInfo head, DatanodeDescriptor dn) {
       int dnIndex = this.findDatanode(dn);
       assert dnIndex >= 0 : "Data node is not found: current";
+      assert getPrevious(dnIndex) == null && getNext(dnIndex) == null : 
+              "Block is already in the list and cannot be inserted.";
       this.setPrevious(dnIndex, null);
       this.setNext(dnIndex, head);
-      if(head != null) {
-        int headDNIndex = head.findDatanode(dn);
-        assert headDNIndex >= 0 : "Data node is not found: head";
-        head.setPrevious(headDNIndex, this);
-      }
+      if(head != null)
+        head.setPrevious(head.findDatanode(dn), this);
       return this;
     }
 
@@ -228,16 +228,10 @@
       BlockInfo prev = this.getPrevious(dnIndex);
       this.setNext(dnIndex, null);
       this.setPrevious(dnIndex, null);
-      if(prev != null) {
-        int prevDNIndex = prev.findDatanode(dn);
-        assert prevDNIndex >= 0 : "Data node is not found: previous";
-        prev.setNext(prevDNIndex, next);
-      }
-      if(next != null) {
-        int nextDNIndex = next.findDatanode(dn);
-        assert nextDNIndex >= 0 : "Data node is not found: next";
-        next.setPrevious(nextDNIndex, prev);
-      }
+      if(prev != null)
+        prev.setNext(prev.findDatanode(dn), next);
+      if(next != null)
+        next.setPrevious(next.findDatanode(dn), prev);
       if(this == head)  // removing the head
         head = next;
       return head;
@@ -250,6 +244,26 @@
         count++;
       return count;
     }
+
+    boolean listIsConsistent(DatanodeDescriptor dn) {
+      // going forward
+      int count = 0;
+      BlockInfo next, nextPrev;
+      BlockInfo cur = this;
+      while(cur != null) {
+        next = cur.getNext(cur.findDatanode(dn));
+        if(next != null) {
+          nextPrev = next.getPrevious(next.findDatanode(dn));
+          if(cur != nextPrev) {
+            System.out.println("Inconsistent list: cur->next->prev != cur");
+            return false;
+          }
+        }
+        cur = next;
+        count++;
+      }
+      return true;
+    }
   }
 
   private static class NodeIterator implements Iterator<DatanodeDescriptor> {
@@ -277,7 +291,8 @@
   private Map<Block, BlockInfo> map = new HashMap<Block, BlockInfo>();
 
   /**
-   * Add BlockInfo if mapping does not exist. */
+   * Add BlockInfo if mapping does not exist.
+   */
   private BlockInfo checkBlockInfo(Block b, int replication) {
     BlockInfo info = map.get(b);
     if (info == null) {
@@ -337,11 +352,8 @@
   boolean addNode(Block b, DatanodeDescriptor node, int replication) {
     // insert into the map if not there yet
     BlockInfo info = checkBlockInfo(b, replication);
-    // add node to the block info
-    boolean added = info.addNode(node);
-    // add to the data-node list
-    node.addBlock(info);
-    return added;
+    // add block to the data-node list and the node to the block info
+    return node.addBlock(info);
   }
 
   /**
@@ -353,10 +365,10 @@
     BlockInfo info = map.get(b);
     if (info == null)
       return false;
-    // first remove block from the data-node list
-    node.removeBlock(info);
-    // remove node from the block info
-    boolean removed = info.removeNode(node);
+
+    // remove block from the data-node list and the node from the block info
+    boolean removed = node.removeBlock(info);
+
     if (info.getDatanode(0) == null     // no datanodes left
               && info.inode == null) {  // does not belong to a file
       map.remove(b);  // remove block from the map

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java?rev=577372&r1=577371&r2=577372&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java Wed Sep 19
09:58:49 2007
@@ -132,17 +132,32 @@
   }
 
   /**
+   * Add data-node to the block.
    * Add block to the head of the list of blocks belonging to the data-node.
    */
-  void addBlock(BlockInfo b) {
+  boolean addBlock(BlockInfo b) {
+    if(!b.addNode(this))
+      return false;
+    // add to the head of the data-node list
     blockList = b.listInsert(blockList, this);
+    return true;
   }
   
   /**
    * Remove block from the list of blocks belonging to the data-node.
+   * Remove data-node from the block.
    */
-  void removeBlock(BlockInfo b) {
+  boolean removeBlock(BlockInfo b) {
     blockList = b.listRemove(blockList, this);
+    return b.removeNode(this);
+  }
+
+  /**
+   * Move block to the head of the list of blocks belonging to the data-node.
+   */
+  void moveBlockToHead(BlockInfo b) {
+    blockList = b.listRemove(blockList, this);
+    blockList = b.listInsert(blockList, this);
   }
 
   void resetBlocks() {
@@ -167,6 +182,9 @@
     this.xceiverCount = xceiverCount;
   }
 
+  /**
+   * Iterates over the list of blocks belonging to the data-node.
+   */
   static private class BlockIterator implements Iterator<Block> {
     private BlockInfo current;
     private DatanodeDescriptor node;
@@ -303,11 +321,14 @@
                   Block[] newReport,
                   Collection<Block> toAdd,
                   Collection<Block> toRemove) {
+    // place a deilimiter in the list which separates blocks 
+    // that have been reported from those that have not
     BlockInfo delimiter = new BlockInfo(new Block(), 1);
-    delimiter.addNode(this);
-    this.addBlock(delimiter); // add to the head of the list
+    boolean added = this.addBlock(delimiter);
+    assert added : "Delimiting block cannot be present in the node";
     if(newReport == null)
       newReport = new Block[0];
+    // scan the report and collect newly reported blocks
     for(Block blk : newReport) {
       BlockInfo storedBlock = blocksMap.getStoredBlock(blk);
       if(storedBlock == null || storedBlock.findDatanode(this) < 0) {
@@ -315,11 +336,10 @@
         continue;
       }
       // move block to the head of the list
-      this.removeBlock(storedBlock);
-      this.addBlock(storedBlock);
+      this.moveBlockToHead(storedBlock);
     }
     // collect blocks that have not been reported
-    // they are all next to the delimiter
+    // all of them are next to the delimiter
     Iterator<Block> it = new BlockIterator(delimiter.getNext(0), this);
     while(it.hasNext())
       toRemove.add(it.next());



Mime
View raw message