hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhr...@apache.org
Subject svn commit: r562572 - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java src/java/org/apache/hadoop/dfs/FSDirectory.java
Date Fri, 03 Aug 2007 20:17:25 GMT
Author: dhruba
Date: Fri Aug  3 13:17:25 2007
New Revision: 562572

URL: http://svn.apache.org/viewvc?view=rev&rev=562572
Log:
HADOOP-1565.  Reduce memory usage of NameNode by replacing
TreeMap in HDFS Namespace with ArrayList.
(Dhruba Borthakur via dhruba)


Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=562572&r1=562571&r2=562572
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Fri Aug  3 13:17:25 2007
@@ -20,11 +20,17 @@
 
   OPTIMIZATIONS
 
+    HADOOP-1565.  Reduce memory usage of NameNode by replacing 
+    TreeMap in HDFS Namespace with ArrayList.  
+    (Dhruba Borthakur via dhruba)
+
   BUG FIXES
 
     HADOOP-1463.  HDFS report correct usage statistics for disk space
     used by HDFS.  (Hairong Kuang via dhruba)
 
+    HADOOP-1666.  FsShell object can be used for multiple fs commands.
+    (Dhruba Borthakur via dhruba)
 
 Branch 0.14 (unreleased changes)
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java?view=diff&rev=562572&r1=562571&r2=562572
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java Fri Aug  3
13:17:25 2007
@@ -19,9 +19,6 @@
 
 import java.util.*;
 
-import org.apache.hadoop.net.NetworkTopology;
-import org.apache.hadoop.net.Node;
-
 /**************************************************
  * DatanodeDescriptor tracks stats on a given DataNode,
  * such as available storage capacity, last update time, etc.,
@@ -172,14 +169,6 @@
     return blocks.keySet().iterator();
   }
   
-  Block getBlock(long blockId) {
-    return blocks.get(new Block(blockId, 0));
-  }
-  
-  Block getBlock(Block b) {
-    return blocks.get(b);
-  }
-  
   /*
    * Store block replication work.
    */
@@ -218,7 +207,7 @@
   }
 
   /*
-   * The number of block invalidattion items that are pending to 
+   * The number of block invalidation items that are pending to 
    * be sent to the datanode
    */
   int getNumberOfBlocksToBeInvalidated() {

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java?view=diff&rev=562572&r1=562571&r2=562572
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java Fri Aug  3 13:17:25
2007
@@ -47,10 +47,11 @@
 
     private String name;
     private INode parent;
-    private TreeMap<String, INode> children = null;
+    private List<INode> children = null;
     private Block blocks[] = null;
     private short blockReplication;
     private long modificationTime;
+    private static final int DEFAULT_FILES_PER_DIRECTORY = 5;
 
     /**
      */
@@ -105,6 +106,13 @@
     }
 
     /**
+     * Set local file name
+     */
+    void setLocalName(String name) {
+      this.name = name;
+    }
+
+    /**
      * Get the full absolute path name of this file (recursively computed).
      * 
      * @return the string representation of the absolute path of this file
@@ -170,15 +178,25 @@
      * @return Iterator of children
      */
     Iterator<INode> getChildIterator() {
-      return (children != null) ?  children.values().iterator() : null;
+      return (children != null) ?  children.iterator() : null;
       // instead of null, we could return a static empty iterator.
     }
         
-    void addChild(String name, INode node) {
+    private void addChild(String name, INode node) {
       if (children == null) {
-        children = new TreeMap<String, INode>();
+        children = new ArrayList<INode>(DEFAULT_FILES_PER_DIRECTORY);
+      }
+      int low = Collections.binarySearch(children, node, comp);
+      assert low < 0;
+      children.add(-low - 1, node);
+    }
+
+    private void removeChild(INode node) {
+      assert children != null;
+      int low = Collections.binarySearch(children, node, comp);
+      if (low >= 0) {
+        children.remove(low);
       }
-      children.put(name, node);
     }
 
     /**
@@ -225,7 +243,37 @@
     }
         
     INode getChild(String name) {
-      return (children == null) ? null : children.get(name);
+      return getChild(new INode(name));
+    }
+
+    private INode getChild(INode node) {
+      if (children == null) {
+        return null;
+      }
+      int low = Collections.binarySearch(children, node, comp);
+      if (low >= 0) {
+        return children.get(low);
+      }
+      return null;
+    }
+
+    //
+    // Prints out the children along with their hash codes.
+    // Useful for debugging.
+    //
+    private void printChildTree(List<INode> children) {
+      System.out.print("Names: ");
+      for (int i = 0; i < children.size(); i++) {
+        INode inode = children.get(i);
+        System.out.print(" " + inode.name);
+      }
+      System.out.println("");
+      System.out.print("Hashcodes: ");
+      for (int i = 0; i < children.size(); i++) {
+        INode inode = children.get(i);
+        System.out.print(" " + inode.name.hashCode());
+      }
+      System.out.println("");
     }
 
     /**
@@ -254,8 +302,9 @@
                                         "Parent path is not a directory: "+path);
       }
       // check whether the parent already has a node with that name
-      String name = newNode.name = target.getName();
-      if (parentNode.getChild(name) != null) {
+      String name =  target.getName();
+      newNode.setLocalName(name);
+      if (parentNode.getChild(newNode) != null) {
         return null;
       }
       // insert into the parent children list
@@ -270,7 +319,7 @@
       if (parent == null) {
         return false;
       } else {
-        parent.children.remove(name);
+        parent.removeChild(this);
         return true;
       }
     }
@@ -349,6 +398,16 @@
         v.add(it.next());
       }
     }
+
+    //
+    // Use the name of the INode to implement a natural order 
+    //
+    static private final Comparator<INode> comp =
+      new Comparator<INode>() {
+        public int compare(INode a, INode b) {
+          return a.getLocalName().compareTo(b.getLocalName());
+        }
+      };
   }
 
   FSNamesystem namesystem = null;



Mime
View raw message