hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject svn commit: r726129 - in /hadoop/core/trunk: ./ src/hdfs/org/apache/hadoop/hdfs/ src/hdfs/org/apache/hadoop/hdfs/protocol/ src/hdfs/org/apache/hadoop/hdfs/server/namenode/ src/hdfs/org/apache/hadoop/hdfs/tools/ src/test/org/apache/hadoop/hdfs/server/na...
Date Fri, 12 Dec 2008 22:02:58 GMT
Author: shv
Date: Fri Dec 12 14:02:58 2008
New Revision: 726129

URL: http://svn.apache.org/viewvc?rev=726129&view=rev
Log:
HADOOP-4826. Introduce admin command saveNamespace. Contributed by Konstantin Shvachko.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java
    hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
    hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
    hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
    hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
    hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java
    hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=726129&r1=726128&r2=726129&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Fri Dec 12 14:02:58 2008
@@ -84,6 +84,8 @@
 
     HADOOP-4348. Add service-level authorization for Hadoop. (acmurthy) 
 
+    HADOOP-4826. Introduce admin command saveNamespace. (shv)
+
   IMPROVEMENTS
 
     HADOOP-4749. Added a new counter REDUCE_INPUT_BYTES. (Yongqiang He via 
@@ -395,7 +397,7 @@
     HADOOP-4727. Fix a group checking bug in fill_stat_structure(...) in
     fuse-dfs.  (Brian Bockelman via szetszwo)
 
-    HADOOP-4836. Correct typos in mapred related documentation.  (Jordà Polo
+    HADOOP-4836. Correct typos in mapred related documentation.  (Jord? Polo
     via szetszwo)
 
 Release 0.19.0 - 2008-11-18

Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java?rev=726129&r1=726128&r2=726129&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java Fri Dec 12 14:02:58 2008
@@ -471,7 +471,7 @@
    * @param progress for reporting write-progress
    * @return an output stream for writing into the file
    * @throws IOException
-   * @see {@link ClientProtocol#append(String, String)}
+   * @see ClientProtocol#append(String, String)
    */
   OutputStream append(String src, int buffersize, Progressable progress
       ) throws IOException {
@@ -773,6 +773,21 @@
   }
 
   /**
+   * Save namespace image.
+   * See {@link ClientProtocol#saveNamespace()} 
+   * for more details.
+   * 
+   * @see ClientProtocol#saveNamespace()
+   */
+  void saveNamespace() throws AccessControlException, IOException {
+    try {
+      namenode.saveNamespace();
+    } catch(RemoteException re) {
+      throw re.unwrapRemoteException(AccessControlException.class);
+    }
+  }
+
+  /**
    * Refresh the hosts and exclude files.  (Rereads them.)
    * See {@link ClientProtocol#refreshNodes()} 
    * for more details.

Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java?rev=726129&r1=726128&r2=726129&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java Fri Dec 12
14:02:58 2008
@@ -33,6 +33,7 @@
 import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.DFSClient.DFSOutputStream;
+import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.util.*;
 
 
@@ -329,7 +330,16 @@
     return dfs.setSafeMode(action);
   }
 
-  /*
+  /**
+   * Save namespace image.
+   * 
+   * @see org.apache.hadoop.hdfs.protocol.ClientProtocol#saveNamespace()
+   */
+  public void saveNamespace() throws AccessControlException, IOException {
+    dfs.saveNamespace();
+  }
+
+  /**
    * Refreshes the list of hosts and excluded hosts from the configured 
    * files.  
    */

Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java?rev=726129&r1=726128&r2=726129&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java Fri Dec
12 14:02:58 2008
@@ -40,9 +40,9 @@
    * Compared to the previous version the following changes have been introduced:
    * (Only the latest change is reflected.
    * The log of historical changes can be retrieved from the svn).
-   * 40: added disk space quotas.
+   * 41: saveNamespace introduced.
    */
-  public static final long versionID = 40L;
+  public static final long versionID = 41L;
   
   ///////////////////////////////////////
   // File contents
@@ -361,6 +361,17 @@
   public boolean setSafeMode(FSConstants.SafeModeAction action) throws IOException;
 
   /**
+   * Save namespace image.
+   * <p>
+   * Saves current namespace into storage directories and reset edits log.
+   * Requires superuser privilege and safe mode.
+   * 
+   * @throws AccessControlException if the superuser privilege is violated.
+   * @throws IOException if image creation failed.
+   */
+  public void saveNamespace() throws IOException;
+
+  /**
    * Tells the namenode to reread the hosts and exclude files. 
    * @throws IOException
    */

Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=726129&r1=726128&r2=726129&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Fri
Dec 12 14:02:58 2008
@@ -1384,7 +1384,7 @@
    * Allocate a block at the given pending filename
    * 
    * @param src path to the file
-   * @param indoes INode representing each of the components of src. 
+   * @param inodes INode representing each of the components of src. 
    *        <code>inodes[inodes.length-1]</code> is the INode for the file.
    */
   private Block allocateBlock(String src, INode[] inodes) throws IOException {
@@ -1764,7 +1764,7 @@
   /**
    * Move a file that is being written to be immutable.
    * @param src The filename
-   * @param holder The datanode that was creating the file
+   * @param lease The lease for the client creating the file
    */
   void internalReleaseLease(Lease lease, String src) throws IOException {
     LOG.info("Recovering lease=" + lease + ", src=" + src);
@@ -3372,7 +3372,25 @@
     }
     return arr;
   }
-    
+
+  /**
+   * Save namespace image.
+   * This will save current namespace into fsimage file and empty edits file.
+   * Requires superuser privilege and safe mode.
+   * 
+   * @throws AccessControlException if superuser privilege is violated.
+   * @throws IOException if 
+   */
+  synchronized void saveNamespace() throws AccessControlException, IOException {
+    checkSuperuserPrivilege();
+    if(!isInSafeMode()) {
+      throw new IOException("Safe mode should be turned ON " +
+                            "in order to create namespace image.");
+    }
+    getFSImage().saveFSImage();
+    LOG.info("New namespace image has been created.");
+  }
+
   /**
    */
   public synchronized void DFSNodesStatus(ArrayList<DatanodeDescriptor> live, 

Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=726129&r1=726128&r2=726129&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java Fri Dec
12 14:02:58 2008
@@ -24,7 +24,8 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.Trash;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.permission.*;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.PermissionStatus;
 import org.apache.hadoop.hdfs.HDFSPolicyProvider;
 import org.apache.hadoop.hdfs.protocol.*;
 import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
@@ -595,7 +596,14 @@
     return namesystem.isInSafeMode();
   }
 
-  /*
+  /**
+   * @inheritDoc
+   */
+  public void saveNamespace() throws IOException {
+    namesystem.saveNamespace();
+  }
+
+  /**
    * Refresh the list of datanodes that the namenode should allow to  
    * connect.  Re-reads conf by creating new Configuration object and 
    * uses the files list in the configuration to update the list. 

Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java?rev=726129&r1=726128&r2=726129&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java Fri Dec 12 14:02:58
2008
@@ -18,7 +18,6 @@
 package org.apache.hadoop.hdfs.tools;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.List;
 
 import javax.security.auth.login.LoginException;
@@ -364,6 +363,27 @@
   }
 
   /**
+   * Command to ask the namenode to save the namespace.
+   * Usage: java DFSAdmin -saveNamespace
+   * @exception IOException 
+   * @see org.apache.hadoop.hdfs.protocol.ClientProtocol#saveNamespace()
+   */
+  public int saveNamespace() throws IOException {
+    int exitCode = -1;
+
+    if (!(fs instanceof DistributedFileSystem)) {
+      System.err.println("FileSystem is " + fs.getUri());
+      return exitCode;
+    }
+
+    DistributedFileSystem dfs = (DistributedFileSystem) fs;
+    dfs.saveNamespace();
+    exitCode = 0;
+   
+    return exitCode;
+  }
+
+  /**
    * Command to ask the namenode to reread the hosts and excluded hosts 
    * file.
    * Usage: java DFSAdmin -refreshNodes
@@ -373,7 +393,7 @@
     int exitCode = -1;
 
     if (!(fs instanceof DistributedFileSystem)) {
-      System.err.println("FileSystem is " + fs.getName());
+      System.err.println("FileSystem is " + fs.getUri());
       return exitCode;
     }
 
@@ -388,6 +408,7 @@
     String summary = "hadoop dfsadmin is the command to execute DFS administrative commands.\n"
+
       "The full syntax is: \n\n" +
       "hadoop dfsadmin [-report] [-safemode <enter | leave | get | wait>]\n" +
+      "\t[-saveNamespace]\n" +
       "\t[-refreshNodes]\n" +
       "\t[" + SetQuotaCommand.USAGE + "]\n" +
       "\t[" + ClearQuotaCommand.USAGE +"]\n" +
@@ -408,6 +429,10 @@
       "\t\tcondition.  Safe mode can also be entered manually, but then\n" +
       "\t\tit can only be turned off manually as well.\n";
 
+    String saveNamespace = "-saveNamespace:\t" +
+    "Save current namespace into storage directories and reset edits log.\n" +
+    "\t\tRequires superuser permissions and safe mode.\n";
+
     String refreshNodes = "-refreshNodes: \tUpdates the set of hosts allowed " +
                           "to connect to namenode.\n\n" +
       "\t\tRe-reads the config file to update values defined by \n" +
@@ -419,7 +444,7 @@
       "\t\tdecommissioning if it has aleady been marked for decommission.\n" + 
       "\t\tEntires not present in both the lists are decommissioned.\n";
 
-    String finalizeUpgrade = "-finalizeUpgrade: Finalize upgrade of DFS.\n" +
+    String finalizeUpgrade = "-finalizeUpgrade: Finalize upgrade of HDFS.\n" +
       "\t\tDatanodes delete their previous version working directories,\n" +
       "\t\tfollowed by Namenode doing the same.\n" + 
       "\t\tThis completes the upgrade process.\n";
@@ -446,6 +471,8 @@
       System.out.println(report);
     } else if ("safemode".equals(cmd)) {
       System.out.println(safemode);
+    } else if ("saveNamespace".equals(cmd)) {
+      System.out.println(saveNamespace);
     } else if ("refreshNodes".equals(cmd)) {
       System.out.println(refreshNodes);
     } else if ("finalizeUpgrade".equals(cmd)) {
@@ -470,6 +497,7 @@
       System.out.println(summary);
       System.out.println(report);
       System.out.println(safemode);
+      System.out.println(saveNamespace);
       System.out.println(refreshNodes);
       System.out.println(finalizeUpgrade);
       System.out.println(upgradeProgress);
@@ -609,6 +637,9 @@
     } else if ("-safemode".equals(cmd)) {
       System.err.println("Usage: java DFSAdmin"
                          + " [-safemode enter | leave | get | wait]");
+    } else if ("-saveNamespace".equals(cmd)) {
+      System.err.println("Usage: java DFSAdmin"
+                         + " [-saveNamespace]");
     } else if ("-refreshNodes".equals(cmd)) {
       System.err.println("Usage: java DFSAdmin"
                          + " [-refreshNodes]");
@@ -640,6 +671,7 @@
       System.err.println("Usage: java DFSAdmin");
       System.err.println("           [-report]");
       System.err.println("           [-safemode enter | leave | get | wait]");
+      System.err.println("           [-saveNamespace]");
       System.err.println("           [-refreshNodes]");
       System.err.println("           [-finalizeUpgrade]");
       System.err.println("           [-upgradeProgress status | details | force]");
@@ -685,6 +717,11 @@
         printUsage(cmd);
         return exitCode;
       }
+    } else if ("-saveNamespace".equals(cmd)) {
+      if (argv.length != 1) {
+        printUsage(cmd);
+        return exitCode;
+      }
     } else if ("-refreshNodes".equals(cmd)) {
       if (argv.length != 1) {
         printUsage(cmd);
@@ -730,6 +767,8 @@
         report();
       } else if ("-safemode".equals(cmd)) {
         setSafeMode(argv, i);
+      } else if ("-saveNamespace".equals(cmd)) {
+        exitCode = saveNamespace();
       } else if ("-refreshNodes".equals(cmd)) {
         exitCode = refreshNodes();
       } else if ("-finalizeUpgrade".equals(cmd)) {

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java?rev=726129&r1=726128&r2=726129&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
(original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
Fri Dec 12 14:02:58 2008
@@ -23,15 +23,18 @@
 import java.util.List;
 import java.util.Iterator;
 import java.util.Random;
+
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.server.common.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction;
 import org.apache.hadoop.hdfs.server.common.Storage;
 import org.apache.hadoop.hdfs.server.namenode.FSImage.NameNodeFile;
 import org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode.ErrorSimulator;
 import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
 import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.namenode.FSImage.NameNodeDirType;
+import org.apache.hadoop.hdfs.tools.DFSAdmin;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
@@ -176,7 +179,6 @@
     // Also check that the edits file is empty here
     // and that temporary checkpoint files are gone.
     FSImage image = cluster.getNameNode().getFSImage();
-    int nrDirs = image.getNumStorageDirs();
     for (Iterator<StorageDirectory> it = 
              image.dirIterator(NameNodeDirType.IMAGE); it.hasNext();) {
       StorageDirectory sd = it.next();
@@ -558,7 +560,7 @@
   }
 
   /**
-   * Tests checkpoint in DFS.
+   * Tests checkpoint in HDFS.
    */
   public void testCheckpoint() throws IOException {
     Path file1 = new Path("checkpoint.dat");
@@ -651,4 +653,62 @@
     testSecondaryFailsToReturnImage(conf);
     testStartup(conf);
   }
+
+  /**
+   * Tests save namepsace.
+   */
+  public void testSaveNamespace() throws IOException {
+    MiniDFSCluster cluster = null;
+    DistributedFileSystem fs = null;
+    try {
+      Configuration conf = new Configuration();
+      cluster = new MiniDFSCluster(conf, numDatanodes, false, null);
+      cluster.waitActive();
+      fs = (DistributedFileSystem)(cluster.getFileSystem());
+
+      // Saving image without safe mode should fail
+      DFSAdmin admin = new DFSAdmin(conf);
+      String[] args = new String[]{"-saveNamespace"};
+      try {
+        admin.run(args);
+      } catch(IOException eIO) {
+        assertTrue(eIO.getLocalizedMessage().contains("Safe mode should be turned ON"));
+      } catch(Exception e) {
+        throw new IOException(e);
+      }
+      // create new file
+      Path file = new Path("namespace.dat");
+      writeFile(fs, file, replication);
+      checkFile(fs, file, replication);
+      // verify that the edits file is NOT empty
+      Collection<File> editsDirs = cluster.getNameEditsDirs();
+      for(File ed : editsDirs) {
+        assertTrue(new File(ed, "current/edits").length() > Integer.SIZE/Byte.SIZE);
+      }
+
+      // Saving image in safe mode should succeed
+      fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+      try {
+        admin.run(args);
+      } catch(Exception e) {
+        throw new IOException(e);
+      }
+      // verify that the edits file is empty
+      for(File ed : editsDirs) {
+        assertTrue(new File(ed, "current/edits").length() == Integer.SIZE/Byte.SIZE);
+      }
+
+      // restart cluster and verify file exists
+      cluster.shutdown();
+      cluster = null;
+
+      cluster = new MiniDFSCluster(conf, numDatanodes, false, null);
+      cluster.waitActive();
+      fs = (DistributedFileSystem)(cluster.getFileSystem());
+      checkFile(fs, file, replication);
+    } finally {
+      if(fs != null) fs.close();
+      if(cluster!= null) cluster.shutdown();
+    }
+  }
 }



Mime
View raw message