hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r488415 - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java src/java/org/apache/hadoop/dfs/DatanodeID.java src/java/org/apache/hadoop/dfs/DatanodeInfo.java src/java/org/apache/hadoop/dfs/FSImage.java
Date Mon, 18 Dec 2006 20:58:53 GMT
Author: cutting
Date: Mon Dec 18 12:58:52 2006
New Revision: 488415

URL: http://svn.apache.org/viewvc?view=rev&rev=488415
Log:
HADOOP-829.  Within HDFS, clearly separate three different representations for datanodes.
 Contributed by Dhruba.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeID.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeInfo.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSImage.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=488415&r1=488414&r2=488415
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Mon Dec 18 12:58:52 2006
@@ -114,6 +114,11 @@
 32. HADOOP-811.  Add a utility, MultithreadedMapRunner.
     (Alejandro Abdelnur via cutting)
 
+33. HADOOP-829.  Within HDFS, clearly separate three different
+    representations for datanodes: one for RPCs, one for
+    namenode-internal use, and one for namespace persistence.
+    (Dhruba Borthakur via cutting) 
+
 
 Release 0.9.2 - 2006-12-15
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java?view=diff&rev=488415&r1=488414&r2=488415
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java Mon Dec 18
12:58:52 2006
@@ -24,6 +24,11 @@
  * such as available storage capacity, last update time, etc.,
  * and maintains a set of blocks stored on the datanode. 
  *
+ * This data structure is a data structure that is internal
+ * to the namenode. It is *not* sent over-the-wire to the Client
+ * or the Datnodes. Neither is it stored persistently in the
+ * fsImage.
+
  * @author Mike Cafarella
  * @author Konstantin Shvachko
  **************************************************/

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeID.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeID.java?view=diff&rev=488415&r1=488414&r2=488415
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeID.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeID.java Mon Dec 18 12:58:52
2006
@@ -68,7 +68,14 @@
   public int getInfoPort() {
     return infoPort;
   }
-  
+
+  /**
+   * @sets data storage ID.
+   */
+  void setStorageID(String storageID) {
+    this.storageID = storageID;
+  }
+
   /**
    * @return hostname and no :portNumber.
    */

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeInfo.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeInfo.java?view=diff&rev=488415&r1=488414&r2=488415
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeInfo.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeInfo.java Mon Dec 18 12:58:52
2006
@@ -30,6 +30,8 @@
 
 /** 
  * DatanodeInfo represents the status of a DataNode.
+ * This object is used for communication in the
+ * Datanode Protocol and the Client Protocol.
  *
  * @author Mike Cafarella
  * @author Konstantin Shvachko
@@ -71,6 +73,26 @@
 
   /** number of active connections */
   public int getXceiverCount() { return xceiverCount; }
+
+  /** Sets raw capacity. */
+  void setCapacity(long capacity) { 
+    this.capacity = capacity; 
+  }
+
+  /** Sets raw free space. */
+  void setRemaining(long remaining) { 
+    this.remaining = remaining; 
+  }
+
+  /** Sets time when this information was accurate. */
+  void setLastUpdate(long lastUpdate) { 
+    this.lastUpdate = lastUpdate; 
+  }
+
+  /** Sets number of active connections */
+  void setXceiverCount(int xceiverCount) { 
+    this.xceiverCount = xceiverCount; 
+  }
 
   /** A formatted string for reporting the status of the DataNode. */
   public String getDatanodeReport() {

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSImage.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSImage.java?view=diff&rev=488415&r1=488414&r2=488415
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSImage.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSImage.java Mon Dec 18 12:58:52 2006
@@ -25,6 +25,8 @@
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.DataInput;
+import java.io.DataOutput;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Random;
@@ -33,6 +35,12 @@
 import org.apache.hadoop.dfs.FSDirectory.INode;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.io.UTF8;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableFactories;
+import org.apache.hadoop.io.WritableFactory;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.UTF8;
 
 /**
  * FSImage handles checkpointing and logging of the namespace edits.
@@ -318,8 +326,10 @@
     Map datanodeMap = FSNamesystem.getFSNamesystem().datanodeMap;
     int size = datanodeMap.size();
     out.writeInt( size );
-    for( Iterator it = datanodeMap.values().iterator(); it.hasNext(); )
-      ((DatanodeDescriptor)it.next()).write( out );
+    for( Iterator it = datanodeMap.values().iterator(); it.hasNext(); ) {
+      DatanodeImage nodeImage = new DatanodeImage((DatanodeDescriptor) it.next());
+      nodeImage.write( out );
+    }
   }
 
   void loadDatanodes( int version, DataInputStream in ) throws IOException {
@@ -328,9 +338,75 @@
     FSNamesystem fsNamesys = FSNamesystem.getFSNamesystem();
     int size = in.readInt();
     for( int i = 0; i < size; i++ ) {
-      DatanodeDescriptor node = new DatanodeDescriptor();
-      node.readFields(in);
-      fsNamesys.unprotectedAddDatanode( node );
+      DatanodeImage nodeImage = new DatanodeImage();
+      nodeImage.readFields(in);
+      fsNamesys.unprotectedAddDatanode(nodeImage.getDatanodeDescriptor());
+    }
+  }
+
+  class DatanodeImage implements WritableComparable {
+
+    /**************************************************
+     * DatanodeImage is used to store persistent information
+     * about datanodes into the fsImage.
+     **************************************************/
+    DatanodeDescriptor              node;
+
+    DatanodeImage() {
+      node = new DatanodeDescriptor();
+    }
+
+    DatanodeImage(DatanodeDescriptor from) {
+      node = from;
+    }
+
+    /** 
+     * Returns the underlying Datanode Descriptor
+     */
+    DatanodeDescriptor getDatanodeDescriptor() { 
+      return node; 
+    }
+
+    public int compareTo(Object o) {
+      return node.compareTo(o);
+    }
+
+    /////////////////////////////////////////////////
+    // Writable
+    /////////////////////////////////////////////////
+    /**
+     * Public method that serializes the information about a
+     * Datanode to be stored in the fsImage.
+     */
+    public void write(DataOutput out) throws IOException {
+      DatanodeID id = new DatanodeID(node.getName(), node.getStorageID(),
+                                     node.getInfoPort());
+      id.write(out);
+      out.writeLong(node.getCapacity());
+      out.writeLong(node.getRemaining());
+      out.writeLong(node.getLastUpdate());
+      out.writeInt(node.getXceiverCount());
+    }
+
+    /**
+     * Public method that reads a serialized Datanode
+     * from the fsImage.
+     */
+    public void readFields(DataInput in) throws IOException {
+      DatanodeID id = new DatanodeID();
+      id.readFields(in);
+      long capacity = in.readLong();
+      long remaining = in.readLong();
+      long lastUpdate = in.readLong();
+      int xceiverCount = in.readInt();
+
+      // update the DatanodeDescriptor with the data we read in
+      node.updateRegInfo(id);
+      node.setStorageID(id.getStorageID());
+      node.setCapacity(capacity);
+      node.setRemaining(remaining);
+      node.setLastUpdate(lastUpdate);
+      node.setXceiverCount(xceiverCount);
     }
   }
 }



Mime
View raw message