hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r677438 - in /hadoop/core/trunk: ./ src/core/org/apache/hadoop/util/ src/docs/src/documentation/content/xdocs/ src/hdfs/org/apache/hadoop/hdfs/server/namenode/ src/hdfs/org/apache/hadoop/hdfs/tools/ src/test/org/apache/hadoop/hdfs/
Date Wed, 16 Jul 2008 21:36:01 GMT
Author: cdouglas
Date: Wed Jul 16 14:36:01 2008
New Revision: 677438

URL: http://svn.apache.org/viewvc?rev=677438&view=rev
Log:
HADOOP-3720. Re-read the config file when dfsadmin -refreshNodes is invoked
so dfs.hosts and dfs.hosts.exclude are observed. Contributed by lohit vijayarenu.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/core/org/apache/hadoop/util/HostsFileReader.java
    hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_design.xml
    hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_user_guide.xml
    hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
    hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
    hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java
    hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestDecommission.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=677438&r1=677437&r2=677438&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed Jul 16 14:36:01 2008
@@ -120,6 +120,10 @@
     HADOOP-3721. Refactor CompositeRecordReader and related mapred.join classes
     to make them clearer. (cdouglas)
 
+    HADOOP-3720. Re-read the config file when dfsadmin -refreshNodes is invoked
+    so dfs.hosts and dfs.hosts.exclude are observed. (lohit vijayarenu via
+    cdouglas)
+
 Release 0.18.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/util/HostsFileReader.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/util/HostsFileReader.java?rev=677438&r1=677437&r2=677438&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/util/HostsFileReader.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/util/HostsFileReader.java Wed Jul 16 14:36:01
2008
@@ -63,7 +63,7 @@
     }  
   }
 
-  public void refresh() throws IOException {
+  public synchronized void refresh() throws IOException {
     includes.clear();
     excludes.clear();
     
@@ -83,4 +83,18 @@
     return excludes;
   }
 
+  public synchronized void setIncludesFile(String includesFile) {
+    this.includesFile = includesFile;
+  }
+  
+  public synchronized void setExcludesFile(String excludesFile) {
+    this.excludesFile = excludesFile;
+  }
+
+  public synchronized void updateFileNames(String includesFile, 
+                                           String excludesFile) 
+                                           throws IOException {
+    setIncludesFile(includesFile);
+    setExcludesFile(excludesFile);
+  }
 }

Modified: hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_design.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_design.xml?rev=677438&r1=677437&r2=677438&view=diff
==============================================================================
--- hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_design.xml (original)
+++ hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_design.xml Wed Jul 16
14:36:01 2008
@@ -313,7 +313,7 @@
             <td> Generate a list of DataNodes </td> <td> <code>bin/hadoop
dfsadmin -report</code> </td>
           </tr>
           <tr>
-            <td> Decommission DataNode <code>datanodename</code> </td><td>
<code>bin/hadoop dfsadmin -decommission datanodename</code> </td>
+            <td> Recommission or decommission DataNode(s) </td><td> <code>bin/hadoop
dfsadmin -refreshNodes</code> </td>
           </tr>
         </table>
       </section>

Modified: hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_user_guide.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_user_guide.xml?rev=677438&r1=677437&r2=677438&view=diff
==============================================================================
--- hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_user_guide.xml (original)
+++ hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_user_guide.xml Wed Jul
16 14:36:01 2008
@@ -190,6 +190,17 @@
    		<code>-finalizeUpgrade</code>
    		: removes previous backup of the cluster made during last upgrade.
    	</li>
+    <li>
+      <code>-refreshNodes</code>
+      : Updates the set of hosts allowed to connect to namenode.
+      Re-reads the config file to update values defined by dfs.hosts and 
+      dfs.host.exclude and reads the entires (hostnames) in those files. 
+      Each entry not defined in dfs.hosts but in dfs.hosts.exclude 
+      is decommissioned. Each entry defined in dfs.hosts and also in 
+      dfs.host.exclude is stopped from decommissioning if it has aleady 
+      been marked for decommission. Entires not present in both the lists 
+      are decommissioned. 
+    </li>
    	</ul>
    	<p>
    	  For command usage, see <a href="commands_manual.html#dfsadmin">dfsadmin command</a>.

Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=677438&r1=677437&r2=677438&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Wed
Jul 16 14:36:01 2008
@@ -3480,6 +3480,7 @@
   }
 
   /**
+   * Rereads the config to get hosts and exclude list file names.
    * Rereads the files to update the hosts and exclude lists.  It
    * checks if any of the hosts have changed states:
    * 1. Added to hosts  --> no further work needed here.
@@ -3487,8 +3488,14 @@
    * 3. Added to exclude --> start decommission.
    * 4. Removed from exclude --> stop decommission.
    */
-  void refreshNodes() throws IOException {
+  public void refreshNodes(Configuration conf) throws IOException {
     checkSuperuserPrivilege();
+    // Reread the config to get dfs.hosts and dfs.hosts.exclude filenames.
+    // Update the file names and refresh internal includes and excludes list
+    if (conf == null)
+      conf = new Configuration();
+    hostsReader.updateFileNames(conf.get("dfs.hosts",""), 
+                                conf.get("dfs.hosts.exclude", ""));
     hostsReader.refresh();
     synchronized (this) {
       for (Iterator<DatanodeDescriptor> it = datanodeMap.values().iterator();

Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=677438&r1=677437&r2=677438&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java Wed Jul
16 14:36:01 2008
@@ -521,10 +521,11 @@
 
   /*
    * Refresh the list of datanodes that the namenode should allow to  
-   * connect.  Uses the files list in the configuration to update the list. 
+   * connect.  Re-reads conf by creating new Configuration object and 
+   * uses the files list in the configuration to update the list. 
    */
   public void refreshNodes() throws IOException {
-    namesystem.refreshNodes();
+    namesystem.refreshNodes(new Configuration());
   }
 
   /**

Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java?rev=677438&r1=677437&r2=677438&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java Wed Jul 16 14:36:01
2008
@@ -305,9 +305,16 @@
       "\t\tcondition.  Safe mode can also be entered manually, but then\n" +
       "\t\tit can only be turned off manually as well.\n";
 
-    String refreshNodes = "-refreshNodes: \tRe-read the hosts and exclude files to update
the set\n" +
-      "\t\tof Datanodes that are allowed to connect to the Namenode\n" +
-      "\t\tand those that should be decommissioned of recommissioned.\n";
+    String refreshNodes = "-refreshNodes: \tUpdates the set of hosts allowed " +
+                          "to connect to namenode.\n\n" +
+      "\t\tRe-reads the config file to update values defined by \n" +
+      "\t\tdfs.hosts and dfs.host.exclude and reads the \n" +
+      "\t\tentires (hostnames) in those files.\n\n" +
+      "\t\tEach entry not defined in dfs.hosts but in \n" + 
+      "\t\tdfs.hosts.exclude is decommissioned. Each entry defined \n" +
+      "\t\tin dfs.hosts and also in dfs.host.exclude is stopped from \n" +
+      "\t\tdecommissioning if it has aleady been marked for decommission.\n" + 
+      "\t\tEntires not present in both the lists are decommissioned.\n";
 
     String finalizeUpgrade = "-finalizeUpgrade: Finalize upgrade of DFS.\n" +
       "\t\tDatanodes delete their previous version working directories,\n" +

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestDecommission.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestDecommission.java?rev=677438&r1=677437&r2=677438&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestDecommission.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestDecommission.java Wed Jul 16 14:36:01
2008
@@ -28,6 +28,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.protocol.FSConstants.DatanodeReportType;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
@@ -170,7 +171,9 @@
   /*
    * decommission one random node.
    */
-  private String decommissionNode(DFSClient client, 
+  private String decommissionNode(NameNode namenode,
+                                  Configuration conf,
+                                  DFSClient client, 
                                   FileSystem filesys,
                                   FileSystem localFileSys)
     throws IOException {
@@ -195,7 +198,7 @@
     ArrayList<String> nodes = new ArrayList<String>(decommissionedNodes);
     nodes.add(nodename);
     writeConfigFile(localFileSys, excludeFile, nodes);
-    dfs.refreshNodes();
+    namenode.namesystem.refreshNodes(conf);
     return nodename;
   }
 
@@ -303,7 +306,8 @@
                            replicas + " replicas.");
         checkFile(fileSys, file1, replicas);
         printFileLocations(fileSys, file1);
-        String downnode = decommissionNode(client, fileSys, localFileSys);
+        String downnode = decommissionNode(cluster.getNameNode(), conf,
+                                           client, fileSys, localFileSys);
         decommissionedNodes.add(downnode);
         waitNodeState(fileSys, downnode, NodeState.DECOMMISSIONED);
         checkFile(fileSys, file1, replicas, downnode);



Mime
View raw message