hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhr...@apache.org
Subject svn commit: r611385 - in /lucene/hadoop/trunk: ./ src/java/org/apache/hadoop/dfs/ src/test/org/apache/hadoop/dfs/
Date Sat, 12 Jan 2008 08:15:13 GMT
Author: dhruba
Date: Sat Jan 12 00:15:13 2008
New Revision: 611385

URL: http://svn.apache.org/viewvc?rev=611385&view=rev
Log:
HADOOP-2540. fsck reports missing blocks incorrectly. (dhruba)


Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestFileCreation.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=611385&r1=611384&r2=611385&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Sat Jan 12 00:15:13 2008
@@ -428,6 +428,8 @@
 
     HADOOP-2562. globPaths supports {ab,cd}.  (Hairong Kuang via dhruba)
 
+    HADOOP-2540. fsck reports missing blocks incorrectly. (dhruba)
+
 Release 0.15.2 - 2008-01-02
 
   BUG FIXES

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java?rev=611385&r1=611384&r2=611385&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java Sat Jan 12 00:15:13
2008
@@ -215,6 +215,9 @@
 
   private long maxFsObjects = 0;          // maximum number of fs objects
 
+  private long softLimit = LEASE_SOFTLIMIT_PERIOD;
+  private long hardLimit = LEASE_HARDLIMIT_PERIOD;
+
   /**
    * FSNamesystem constructor.
    */
@@ -1115,7 +1118,7 @@
       throw new LeaseExpiredException("No lease on " + src);
     }
     INodeFileUnderConstruction pendingFile = (INodeFileUnderConstruction)file;
-    if (!pendingFile.getClientName().equals(holder)) {
+    if (holder != null && !pendingFile.getClientName().equals(holder)) {
       throw new LeaseExpiredException("Lease mismatch on " + src + " owned by "
           + pendingFile.getClientName() + " but is accessed by " + holder);
     }
@@ -1548,7 +1551,7 @@
      * Returns true if the Hard Limit Timer has expired
      */
     public boolean expiredHardLimit() {
-      if (now() - lastUpdate > LEASE_HARDLIMIT_PERIOD) {
+      if (now() - lastUpdate > hardLimit) {
         return true;
       }
       return false;
@@ -1557,7 +1560,7 @@
      * Returns true if the Soft Limit Timer has expired
      */
     public boolean expiredSoftLimit() {
-      if (now() - lastUpdate > LEASE_SOFTLIMIT_PERIOD) {
+      if (now() - lastUpdate > softLimit) {
         return true;
       }
       return false;
@@ -1704,7 +1707,7 @@
     // will report this block as a missing block because no datanodes have it.
     // Delete this block.
     Block[] blocks = pendingFile.getBlocks();
-    if (blocks != null && blocks.length > 1) {
+    if (blocks != null && blocks.length > 0) {
       Block last = blocks[blocks.length - 1];
       if (last.getNumBytes() == 0) {
           pendingFile.removeBlock(last);
@@ -3917,5 +3920,14 @@
    */
   long getMaxObjects() {
     return maxFsObjects;
+  }
+
+  /**
+   * Used by unit tests to change lease periods
+   */
+  void setLeasePeriod(long softLimit, long hardLimit) {
+    this.softLimit = softLimit;
+    this.hardLimit = hardLimit; 
+    this.lmthread.interrupt();
   }
 }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java?rev=611385&r1=611384&r2=611385&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java Sat Jan 12 00:15:13 2008
@@ -85,7 +85,7 @@
   public static final Log LOG = LogFactory.getLog("org.apache.hadoop.dfs.NameNode");
   public static final Log stateChangeLog = LogFactory.getLog("org.apache.hadoop.dfs.StateChange");
 
-  private FSNamesystem namesystem;
+  FSNamesystem namesystem;
   private Server server;
   private Thread emptier;
   private int handlerCount = 2;

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java?rev=611385&r1=611384&r2=611385&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java Sat Jan 12 00:15:13
2008
@@ -508,4 +508,11 @@
      injectBlocks(i, blocksToInject[i]);
     }
   }
+
+  /**
+   * Set the softLimit and hardLimit of client lease periods
+   */
+  void setLeasePeriod(long soft, long hard) {
+    nameNode.namesystem.setLeasePeriod(soft, hard);
+  }
 }

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestFileCreation.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestFileCreation.java?rev=611385&r1=611384&r2=611385&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestFileCreation.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestFileCreation.java Sat Jan 12 00:15:13
2008
@@ -23,10 +23,8 @@
 import java.util.Random;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FsShell;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.dfs.FSConstants.DatanodeReportType;
@@ -47,16 +45,11 @@
   // entire file is written, the first two blocks definitely get flushed to
   // the datanodes.
 
-  private static String TEST_ROOT_DIR =
-    new Path(System.getProperty("test.build.data","/tmp"))
-    .toString().replace(' ', '+');
-  
   //
   // creates a file but does not close it
   //
   private FSDataOutputStream createFile(FileSystem fileSys, Path name, int repl)
     throws IOException {
-    // create and write a file that contains three blocks of data
     FSDataOutputStream stm = fileSys.create(name, true,
                                             fileSys.getConf().getInt("io.file.buffer.size",
4096),
                                             (short)repl, (long)blockSize);
@@ -116,9 +109,9 @@
 
   private void checkData(byte[] actual, int from, byte[] expected, String message) {
     for (int idx = 0; idx < actual.length; idx++) {
-      this.assertEquals(message+" byte "+(from+idx)+" differs. expected "+
-                        expected[from+idx]+" actual "+actual[idx],
-                        expected[from+idx], actual[idx]);
+      assertEquals(message+" byte "+(from+idx)+" differs. expected "+
+                   expected[from+idx]+" actual "+actual[idx],
+                   expected[from+idx], actual[idx]);
       actual[idx] = 0;
     }
   }
@@ -262,6 +255,78 @@
                  locations.locatedBlockCount() == 0);
     } finally {
       fs.close();
+      cluster.shutdown();
+    }
+  }
+
+  /**
+   * Test that the filesystem removes the last block from a file if its
+   * lease expires.
+   */
+  public void testFileCreationError2() throws IOException {
+    long leasePeriod = 1000;
+    System.out.println("testFileCreationError2 start");
+    Configuration conf = new Configuration();
+    conf.setInt("heartbeat.recheck.interval", 1000);
+    conf.setInt("dfs.heartbeat.interval", 1);
+    if (simulatedStorage) {
+      conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true);
+    }
+    // create cluster
+    MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null);
+    FileSystem fs = cluster.getFileSystem();
+    cluster.waitActive();
+    InetSocketAddress addr = new InetSocketAddress("localhost",
+                                                   cluster.getNameNodePort());
+    DFSClient client = new DFSClient(addr, conf);
+
+    try {
+
+      // create a new file.
+      //
+      Path file1 = new Path("/filestatus.dat");
+      createFile(fs, file1, 1);
+      System.out.println("testFileCreationError2: "
+                         + "Created file filestatus.dat with one "
+                         + " replicas.");
+
+      LocatedBlocks locations = client.namenode.getBlockLocations(
+                                  file1.toString(), 0, Long.MAX_VALUE);
+      System.out.println("The file has " + locations.locatedBlockCount() +
+                         " blocks.");
+
+      // add another block to the file
+      LocatedBlock location = client.namenode.addBlock(file1.toString(), 
+                                                       null);
+      System.out.println("Added block " + location.getBlock());
+
+      locations = client.namenode.getBlockLocations(file1.toString(), 
+                                                    0, Long.MAX_VALUE);
+      System.out.println("The file now has " + locations.locatedBlockCount() +
+                         " blocks.");
+      
+      // set the soft and hard limit to be 1 second so that the
+      // namenode triggers lease recovery
+      cluster.setLeasePeriod(leasePeriod, leasePeriod);
+
+      // wait for the lease to expire
+      try {
+        Thread.sleep(5 * leasePeriod);
+      } catch (InterruptedException e) {
+      }
+
+      // verify that the last block was cleaned up.
+      locations = client.namenode.getBlockLocations(file1.toString(), 
+                                                    0, Long.MAX_VALUE);
+      System.out.println("locations = " + locations.locatedBlockCount());
+      assertTrue("Error blocks were not cleaned up",
+                 locations.locatedBlockCount() == 0);
+      System.out.println("testFileCreationError2 successful");
+    } finally {
+      try {
+        fs.close();
+      } catch (Exception e) {
+      }
       cluster.shutdown();
     }
   }



Mime
View raw message