hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r389566 - in /lucene/hadoop/trunk/src: java/org/apache/hadoop/dfs/FSDirectory.java java/org/apache/hadoop/dfs/NameNode.java java/overview.html test/org/apache/hadoop/dfs/MiniDFSCluster.java
Date Tue, 28 Mar 2006 18:14:03 GMT
Author: cutting
Date: Tue Mar 28 10:14:02 2006
New Revision: 389566

URL: http://svn.apache.org/viewcvs?rev=389566&view=rev
Log:
Fix for HADOOP-19.  A namenode must now be formatted before it may be used.  Attempts to start
a namenode in an unformatted directory will fail, rather than automatically creating a new,
empty filesystem, causing existing datanodes to delete all blocks.  Thus a mis-configured
dfs.data.dir should no longer cause data loss.

Modified:
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java
    lucene/hadoop/trunk/src/java/overview.html
    lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java?rev=389566&r1=389565&r2=389566&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java Tue Mar 28 10:14:02
2006
@@ -245,14 +245,11 @@
     DataOutputStream editlog = null;
     boolean ready = false;
 
-    /**
-     * Create a FileSystem directory, and load its info
-     * from the indicated place.
-     */
+    /** Access an existing dfs name directory. */
     public FSDirectory(File dir) throws IOException {
         File fullimage = new File(dir, "image");
         if (! fullimage.exists()) {
-            fullimage.mkdirs();
+          throw new IOException("NameNode not formatted: " + dir);
         }
         File edits = new File(dir, "edits");
         if (loadFSImage(fullimage, edits)) {
@@ -263,6 +260,19 @@
             this.ready = true;
             this.notifyAll();
             this.editlog = new DataOutputStream(new FileOutputStream(edits));
+        }
+    }
+
+    /** Create a new dfs name directory.  Caution: this destroys all files
+     * in this filesystem. */
+    public static void format(File dir) throws IOException {
+        File image = new File(dir, "image");
+        File edits = new File(dir, "edits");
+
+        if (!((!image.exists() || image.delete()) &&
+              (!edits.exists() || edits.delete()) &&
+              image.mkdirs())) {
+          throw new IOException("Unable to format: "+dir);
         }
     }
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java?rev=389566&r1=389565&r2=389566&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java Tue Mar 28 10:14:02 2006
@@ -65,12 +65,17 @@
     /** only used for testing purposes  */
     private boolean stopRequested = false;
 
+    /** Format a new filesystem.  Destroys any filesystem that may already
+     * exist at this location.  **/
+    public static void format(Configuration conf) throws IOException {
+      FSDirectory.format(getDir(conf));
+    }
+
     /**
      * Create a NameNode at the default location
      */
     public NameNode(Configuration conf) throws IOException {
-        this(new File(conf.get("dfs.name.dir",
-                                          "/tmp/hadoop/dfs/name")),
+        this(getDir(conf),
              DataNode.createSocketAddr
              (conf.get("fs.default.name", "local")).getPort(), conf);
     }
@@ -85,6 +90,11 @@
         this.server.start();
     }
 
+    /** Return the configured directory where name data is stored. */
+    private static File getDir(Configuration conf) {
+      return new File(conf.get("dfs.name.dir", "/tmp/hadoop/dfs/name"));
+    }
+
     /**
      * Wait for service to finish.
      * (Normally, it runs forever.)
@@ -364,8 +374,24 @@
 
     /**
      */
-    public static void main(String argv[]) throws IOException, InterruptedException {
-        NameNode namenode = new NameNode(new Configuration());
+    public static void main(String argv[]) throws Exception {
+        Configuration conf = new Configuration();
+
+        if (argv.length == 1 && argv[0].equals("-format")) {
+          File dir = getDir(conf);
+          if (dir.exists()) {
+            System.err.print("Re-format filesystem in " + dir +" ? (Y or N) ");
+            if (!(System.in.read() == 'Y')) {
+              System.err.println("Format aborted.");
+              System.exit(1);
+            }
+          }
+          format(conf);
+          System.err.println("Formatted "+dir);
+          System.exit(0);
+        }
+
+        NameNode namenode = new NameNode(conf);
         namenode.join();
     }
 }

Modified: lucene/hadoop/trunk/src/java/overview.html
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/overview.html?rev=389566&r1=389565&r2=389566&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/overview.html (original)
+++ lucene/hadoop/trunk/src/java/overview.html Tue Mar 28 10:14:02 2006
@@ -134,6 +134,11 @@
 
 <h3>Bootstrapping</h3>
 
+<p>A new distributed filesystem must formatted with the following
+command, run on the master node:</p>
+
+<p><tt>bin/hadoop namenode -format</tt></p>
+
 <p>The Hadoop daemons are started with the following command:</p>
 
 <p><tt>bin/start-all.sh</tt></p>

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java?rev=389566&r1=389565&r2=389566&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java Tue Mar 28 10:14:02
2006
@@ -83,7 +83,7 @@
   /**
    * Create the config and start up the servers.
    */
-  public MiniDFSCluster(int namenodePort, Configuration conf) {
+  public MiniDFSCluster(int namenodePort, Configuration conf) throws IOException {
     this.conf = conf;
     conf.set("fs.default.name", 
              "localhost:"+ Integer.toString(namenodePort));
@@ -95,6 +95,7 @@
     // this timeout seems to control the minimum time for the test, so
     // set it down at 5 seconds.
     conf.setInt("ipc.client.timeout", 5000);
+    NameNode.format(conf);
     nameNode = new NameNodeRunner();
     nameNodeThread = new Thread(nameNode);
     nameNodeThread.start();



Mime
View raw message