hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject svn commit: r1471229 - in /hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/ src/main/java/org/apache/hadoop/hdfs/server/namenode/ src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/ src/main/native/ src/mai...
Date Wed, 24 Apr 2013 02:21:59 GMT
Author: szetszwo
Date: Wed Apr 24 02:21:58 2013
New Revision: 1471229

URL: http://svn.apache.org/r1471229
Log:
Merge r1470760 through r1471228 from trunk.

Modified:
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/   (props changed)
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/   (props
changed)
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsBinaryLoader.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/   (props
changed)
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
  (props changed)
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
  (props changed)
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
  (props changed)
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/   (props
changed)
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-22-dfs-dir.tgz

Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1470760-1471228

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Apr 24
02:21:58 2013
@@ -326,6 +326,10 @@ Trunk (Unreleased)
     HDFS-4646. createNNProxyWithClientProtocol ignores configured timeout
     value (Jagane Sundar via cos)
 
+    HDFS-4732. Fix TestDFSUpgradeFromImage which fails on Windows due to
+    failure to unpack old image tarball that contains hard links.
+    (Chris Nauroth via szetszwo)
+
   BREAKDOWN OF HADOOP-8562 SUBTASKS AND RELATED JIRAS
 
     HDFS-4145. Merge hdfs cmd line scripts from branch-1-win. (David Lao,
@@ -360,6 +364,9 @@ Trunk (Unreleased)
     HDFS-4674. TestBPOfferService fails on Windows due to failure parsing 
     datanode data directory as URI. (Chris Nauroth via suresh)
 
+    HDFS-4725. Fix HDFS file handle leaks in FSEditLog, NameNode,
+    OfflineEditsBinaryLoader and some tests.  (Chris Nauroth via szetszwo)
+
   BREAKDOWN OF HDFS-347 SUBTASKS AND RELATED JIRAS
 
     HDFS-4353. Encapsulate connections to peers in Peer and PeerServer classes.
@@ -573,6 +580,8 @@ Release 2.0.5-beta - UNRELEASED
     HDFS-4695. TestEditLog leaks open file handles between tests.
     (Ivan Mitic via suresh)
 
+    HDFS-4737. JVM path embedded in fuse binaries. (Sean Mackrory via atm)
+
 Release 2.0.4-alpha - UNRELEASED
 
   INCOMPATIBLE CHANGES

Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1470760-1471228

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
(original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
Wed Apr 24 02:21:58 2013
@@ -322,21 +322,23 @@ public class FSEditLog implements LogsPu
       LOG.debug("Closing log when already closed");
       return;
     }
-    if (state == State.IN_SEGMENT) {
-      assert editLogStream != null;
-      waitForSyncToFinish();
-      endCurrentLogSegment(true);
-    }
-    
-    if (journalSet != null && !journalSet.isEmpty()) {
-      try {
-        journalSet.close();
-      } catch (IOException ioe) {
-        LOG.warn("Error closing journalSet", ioe);
+
+    try {
+      if (state == State.IN_SEGMENT) {
+        assert editLogStream != null;
+        waitForSyncToFinish();
+        endCurrentLogSegment(true);
       }
+    } finally {
+      if (journalSet != null && !journalSet.isEmpty()) {
+        try {
+          journalSet.close();
+        } catch (IOException ioe) {
+          LOG.warn("Error closing journalSet", ioe);
+        }
+      }
+      state = State.CLOSED;
     }
-
-    state = State.CLOSED;
   }
 
 
@@ -588,6 +590,7 @@ public class FSEditLog implements LogsPu
                 "due to " + e.getMessage() + ". " +
                 "Unsynced transactions: " + (txid - synctxid);
             LOG.fatal(msg, new Exception());
+            IOUtils.cleanup(LOG, journalSet);
             terminate(1, msg);
           }
         } finally {
@@ -611,6 +614,7 @@ public class FSEditLog implements LogsPu
               "Could not sync enough journals to persistent storage. "
               + "Unsynced transactions: " + (txid - synctxid);
           LOG.fatal(msg, new Exception());
+          IOUtils.cleanup(LOG, journalSet);
           terminate(1, msg);
         }
       }

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
(original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
Wed Apr 24 02:21:58 2013
@@ -651,13 +651,14 @@ public class NameNode {
       }
     } catch (ServiceFailedException e) {
       LOG.warn("Encountered exception while exiting state ", e);
-    }
-    stopCommonServices();
-    if (metrics != null) {
-      metrics.shutdown();
-    }
-    if (namesystem != null) {
-      namesystem.shutdown();
+    } finally {
+      stopCommonServices();
+      if (metrics != null) {
+        metrics.shutdown();
+      }
+      if (namesystem != null) {
+        namesystem.shutdown();
+      }
     }
   }
 

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsBinaryLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsBinaryLoader.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsBinaryLoader.java
(original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsBinaryLoader.java
Wed Apr 24 02:21:58 2013
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdfs.tools.offl
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp;
 
 import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream;
+import org.apache.hadoop.io.IOUtils;
 
 /**
  * OfflineEditsBinaryLoader loads edits from a binary edits file
@@ -59,43 +60,49 @@ class OfflineEditsBinaryLoader implement
    */
   @Override
   public void loadEdits() throws IOException {
-    visitor.start(inputStream.getVersion());
-    while (true) {
-      try {
-        FSEditLogOp op = inputStream.readOp();
-        if (op == null)
-          break;
-        if (fixTxIds) {
-          if (nextTxId <= 0) {
-            nextTxId = op.getTransactionId();
+    try {
+      visitor.start(inputStream.getVersion());
+      while (true) {
+        try {
+          FSEditLogOp op = inputStream.readOp();
+          if (op == null)
+            break;
+          if (fixTxIds) {
             if (nextTxId <= 0) {
-              nextTxId = 1;
+              nextTxId = op.getTransactionId();
+              if (nextTxId <= 0) {
+                nextTxId = 1;
+              }
             }
+            op.setTransactionId(nextTxId);
+            nextTxId++;
           }
-          op.setTransactionId(nextTxId);
-          nextTxId++;
-        }
-        visitor.visitOp(op);
-      } catch (IOException e) {
-        if (!recoveryMode) {
-          // Tell the visitor to clean up, then re-throw the exception
-          LOG.error("Got IOException at position " + inputStream.getPosition());
-          visitor.close(e);
-          throw e;
-        }
-        LOG.error("Got IOException while reading stream!  Resyncing.", e);
-        inputStream.resync();
-      } catch (RuntimeException e) {
-        if (!recoveryMode) {
-          // Tell the visitor to clean up, then re-throw the exception
-          LOG.error("Got RuntimeException at position " + inputStream.getPosition());
-          visitor.close(e);
-          throw e;
+          visitor.visitOp(op);
+        } catch (IOException e) {
+          if (!recoveryMode) {
+            // Tell the visitor to clean up, then re-throw the exception
+            LOG.error("Got IOException at position " +
+              inputStream.getPosition());
+            visitor.close(e);
+            throw e;
+          }
+          LOG.error("Got IOException while reading stream!  Resyncing.", e);
+          inputStream.resync();
+        } catch (RuntimeException e) {
+          if (!recoveryMode) {
+            // Tell the visitor to clean up, then re-throw the exception
+            LOG.error("Got RuntimeException at position " +
+              inputStream.getPosition());
+            visitor.close(e);
+            throw e;
+          }
+          LOG.error("Got RuntimeException while reading stream!  Resyncing.", e);
+          inputStream.resync();
         }
-        LOG.error("Got RuntimeException while reading stream!  Resyncing.", e);
-        inputStream.resync();
       }
+      visitor.close(null);
+    } finally {
+      IOUtils.cleanup(LOG, inputStream);
     }
-    visitor.close(null);
   }
-}
\ No newline at end of file
+}

Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1470760-1471228

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt
(original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt
Wed Apr 24 02:21:58 2013
@@ -16,6 +16,8 @@
 # limitations under the License.
 #
 
+set(CMAKE_SKIP_RPATH TRUE)
+
 # Find Linux FUSE
 IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
     find_package(PkgConfig REQUIRED)

Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1470760-1471228

Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1470760-1471228

Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1470760-1471228

Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1470760-1471228

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
(original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
Wed Apr 24 02:21:58 2013
@@ -48,6 +48,8 @@ import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.TimeoutException;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
@@ -91,6 +93,8 @@ import com.google.common.base.Joiner;
 
 /** Utilities for HDFS tests */
 public class DFSTestUtil {
+
+  private static final Log LOG = LogFactory.getLog(DFSTestUtil.class);
   
   private static Random gen = new Random();
   private static String[] dirNames = {
@@ -742,7 +746,11 @@ public class DFSTestUtil {
     File file = new File(filename);
     DataInputStream in = new DataInputStream(new FileInputStream(file));
     byte[] content = new byte[(int)file.length()];
-    in.readFully(content);
+    try {
+      in.readFully(content);
+    } finally {
+      IOUtils.cleanup(LOG, in);
+    }
     return content;
   }
 

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
(original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
Wed Apr 24 02:21:58 2013
@@ -252,8 +252,9 @@ public class TestDFSUpgradeFromImage {
 
     // Now try to start an NN from it
 
+    MiniDFSCluster cluster = null;
     try {
-      new MiniDFSCluster.Builder(conf).numDataNodes(0)
+      cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
         .format(false)
         .manageDataDfsDirs(false)
         .manageNameDfsDirs(false)
@@ -264,6 +265,11 @@ public class TestDFSUpgradeFromImage {
       if (!ioe.toString().contains("Old layout version is 'too old'")) {
         throw ioe;
       }
+    } finally {
+      // We expect startup to fail, but just in case it didn't, shutdown now.
+      if (cluster != null) {
+        cluster.shutdown();
+      }
     }
   }
   

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
(original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
Wed Apr 24 02:21:58 2013
@@ -631,44 +631,48 @@ public class TestDistributedFileSystem {
         true);
     final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
         .numDataNodes(2).build();
-    DistributedFileSystem fs = cluster.getFileSystem();
-    // Create two files
-    Path tmpFile1 = new Path("/tmpfile1.dat");
-    Path tmpFile2 = new Path("/tmpfile2.dat");
-    DFSTestUtil.createFile(fs, tmpFile1, 1024, (short) 2, 0xDEADDEADl);
-    DFSTestUtil.createFile(fs, tmpFile2, 1024, (short) 2, 0xDEADDEADl);
-    // Get locations of blocks of both files and concat together
-    BlockLocation[] blockLocs1 = fs.getFileBlockLocations(tmpFile1, 0, 1024);
-    BlockLocation[] blockLocs2 = fs.getFileBlockLocations(tmpFile2, 0, 1024);
-    BlockLocation[] blockLocs = (BlockLocation[]) ArrayUtils.addAll(blockLocs1,
-        blockLocs2);
-    // Fetch VolumeBlockLocations in batch
-    BlockStorageLocation[] locs = fs.getFileBlockStorageLocations(Arrays
-        .asList(blockLocs));
-    int counter = 0;
-    // Print out the list of ids received for each block
-    for (BlockStorageLocation l : locs) {
-      for (int i = 0; i < l.getVolumeIds().length; i++) {
-        VolumeId id = l.getVolumeIds()[i];
-        String name = l.getNames()[i];
-        if (id != null) {
-          System.out.println("Datanode " + name + " has block " + counter
-              + " on volume id " + id.toString());
+    try {
+      DistributedFileSystem fs = cluster.getFileSystem();
+      // Create two files
+      Path tmpFile1 = new Path("/tmpfile1.dat");
+      Path tmpFile2 = new Path("/tmpfile2.dat");
+      DFSTestUtil.createFile(fs, tmpFile1, 1024, (short) 2, 0xDEADDEADl);
+      DFSTestUtil.createFile(fs, tmpFile2, 1024, (short) 2, 0xDEADDEADl);
+      // Get locations of blocks of both files and concat together
+      BlockLocation[] blockLocs1 = fs.getFileBlockLocations(tmpFile1, 0, 1024);
+      BlockLocation[] blockLocs2 = fs.getFileBlockLocations(tmpFile2, 0, 1024);
+      BlockLocation[] blockLocs = (BlockLocation[]) ArrayUtils.addAll(blockLocs1,
+          blockLocs2);
+      // Fetch VolumeBlockLocations in batch
+      BlockStorageLocation[] locs = fs.getFileBlockStorageLocations(Arrays
+          .asList(blockLocs));
+      int counter = 0;
+      // Print out the list of ids received for each block
+      for (BlockStorageLocation l : locs) {
+        for (int i = 0; i < l.getVolumeIds().length; i++) {
+          VolumeId id = l.getVolumeIds()[i];
+          String name = l.getNames()[i];
+          if (id != null) {
+            System.out.println("Datanode " + name + " has block " + counter
+                + " on volume id " + id.toString());
+          }
         }
+        counter++;
       }
-      counter++;
-    }
-    assertEquals("Expected two HdfsBlockLocations for two 1-block files", 2,
-        locs.length);
-    for (BlockStorageLocation l : locs) {
-      assertEquals("Expected two replicas for each block", 2,
-          l.getVolumeIds().length);
-      for (int i = 0; i < l.getVolumeIds().length; i++) {
-        VolumeId id = l.getVolumeIds()[i];
-        String name = l.getNames()[i];
-        assertTrue("Expected block to be valid on datanode " + name,
-            id.isValid());
+      assertEquals("Expected two HdfsBlockLocations for two 1-block files", 2,
+          locs.length);
+      for (BlockStorageLocation l : locs) {
+        assertEquals("Expected two replicas for each block", 2,
+            l.getVolumeIds().length);
+        for (int i = 0; i < l.getVolumeIds().length; i++) {
+          VolumeId id = l.getVolumeIds()[i];
+          String name = l.getNames()[i];
+          assertTrue("Expected block to be valid on datanode " + name,
+              id.isValid());
+        }
       }
+    } finally {
+      cluster.shutdown();
     }
   }
 
@@ -683,27 +687,31 @@ public class TestDistributedFileSystem {
         true);
     final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
         .numDataNodes(2).build();
-    cluster.getDataNodes();
-    DistributedFileSystem fs = cluster.getFileSystem();
-    // Create a file
-    Path tmpFile = new Path("/tmpfile1.dat");
-    DFSTestUtil.createFile(fs, tmpFile, 1024, (short) 2, 0xDEADDEADl);
-    // Get locations of blocks of the file
-    BlockLocation[] blockLocs = fs.getFileBlockLocations(tmpFile, 0, 1024);
-    // Stop a datanode to simulate a failure
-    cluster.stopDataNode(0);
-    // Fetch VolumeBlockLocations
-    BlockStorageLocation[] locs = fs.getFileBlockStorageLocations(Arrays
-        .asList(blockLocs));
-
-    assertEquals("Expected one HdfsBlockLocation for one 1-block file", 1,
-        locs.length);
-
-    for (BlockStorageLocation l : locs) {
-      assertEquals("Expected two replicas for each block", 2,
-          l.getVolumeIds().length);
-      assertTrue("Expected one valid and one invalid replica",
-          (l.getVolumeIds()[0].isValid()) ^ (l.getVolumeIds()[1].isValid()));
+    try {
+      cluster.getDataNodes();
+      DistributedFileSystem fs = cluster.getFileSystem();
+      // Create a file
+      Path tmpFile = new Path("/tmpfile1.dat");
+      DFSTestUtil.createFile(fs, tmpFile, 1024, (short) 2, 0xDEADDEADl);
+      // Get locations of blocks of the file
+      BlockLocation[] blockLocs = fs.getFileBlockLocations(tmpFile, 0, 1024);
+      // Stop a datanode to simulate a failure
+      cluster.stopDataNode(0);
+      // Fetch VolumeBlockLocations
+      BlockStorageLocation[] locs = fs.getFileBlockStorageLocations(Arrays
+          .asList(blockLocs));
+
+      assertEquals("Expected one HdfsBlockLocation for one 1-block file", 1,
+          locs.length);
+
+      for (BlockStorageLocation l : locs) {
+        assertEquals("Expected two replicas for each block", 2,
+            l.getVolumeIds().length);
+        assertTrue("Expected one valid and one invalid replica",
+            (l.getVolumeIds()[0].isValid()) ^ (l.getVolumeIds()[1].isValid()));
+      }
+    } finally {
+      cluster.shutdown();
     }
   }
 

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
(original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
Wed Apr 24 02:21:58 2013
@@ -781,49 +781,53 @@ public class TestQuota {
   public void testMaxSpaceQuotas() throws Exception {
     final Configuration conf = new HdfsConfiguration();
     final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
-    final FileSystem fs = cluster.getFileSystem();
-    assertTrue("Not a HDFS: "+fs.getUri(),
-                fs instanceof DistributedFileSystem);
-    final DistributedFileSystem dfs = (DistributedFileSystem)fs;
-    
-    // create test directory
-    final Path testFolder = new Path("/testFolder");
-    assertTrue(dfs.mkdirs(testFolder));
-    
-    // setting namespace quota to Long.MAX_VALUE - 1 should work
-    dfs.setQuota(testFolder, Long.MAX_VALUE - 1, 10);
-    ContentSummary c = dfs.getContentSummary(testFolder);
-    assertTrue("Quota not set properly", c.getQuota() == Long.MAX_VALUE - 1);
-    
-    // setting diskspace quota to Long.MAX_VALUE - 1 should work
-    dfs.setQuota(testFolder, 10, Long.MAX_VALUE - 1);
-    c = dfs.getContentSummary(testFolder);
-    assertTrue("Quota not set properly", c.getSpaceQuota() == Long.MAX_VALUE - 1);
-    
-    // setting namespace quota to Long.MAX_VALUE should not work + no error
-    dfs.setQuota(testFolder, Long.MAX_VALUE, 10);
-    c = dfs.getContentSummary(testFolder);
-    assertTrue("Quota should not have changed", c.getQuota() == 10);
-    
-    // setting diskspace quota to Long.MAX_VALUE should not work + no error
-    dfs.setQuota(testFolder, 10, Long.MAX_VALUE);
-    c = dfs.getContentSummary(testFolder);
-    assertTrue("Quota should not have changed", c.getSpaceQuota() == 10);
-    
-    // setting namespace quota to Long.MAX_VALUE + 1 should not work + error
     try {
-      dfs.setQuota(testFolder, Long.MAX_VALUE + 1, 10);
-      fail("Exception not thrown");
-    } catch (IllegalArgumentException e) {
-      // Expected
-    }
+      final FileSystem fs = cluster.getFileSystem();
+      assertTrue("Not a HDFS: "+fs.getUri(),
+                  fs instanceof DistributedFileSystem);
+      final DistributedFileSystem dfs = (DistributedFileSystem)fs;
+    
+      // create test directory
+      final Path testFolder = new Path("/testFolder");
+      assertTrue(dfs.mkdirs(testFolder));
+    
+      // setting namespace quota to Long.MAX_VALUE - 1 should work
+      dfs.setQuota(testFolder, Long.MAX_VALUE - 1, 10);
+      ContentSummary c = dfs.getContentSummary(testFolder);
+      assertTrue("Quota not set properly", c.getQuota() == Long.MAX_VALUE - 1);
+    
+      // setting diskspace quota to Long.MAX_VALUE - 1 should work
+      dfs.setQuota(testFolder, 10, Long.MAX_VALUE - 1);
+      c = dfs.getContentSummary(testFolder);
+      assertTrue("Quota not set properly", c.getSpaceQuota() == Long.MAX_VALUE - 1);
+    
+      // setting namespace quota to Long.MAX_VALUE should not work + no error
+      dfs.setQuota(testFolder, Long.MAX_VALUE, 10);
+      c = dfs.getContentSummary(testFolder);
+      assertTrue("Quota should not have changed", c.getQuota() == 10);
+    
+      // setting diskspace quota to Long.MAX_VALUE should not work + no error
+      dfs.setQuota(testFolder, 10, Long.MAX_VALUE);
+      c = dfs.getContentSummary(testFolder);
+      assertTrue("Quota should not have changed", c.getSpaceQuota() == 10);
     
-    // setting diskspace quota to Long.MAX_VALUE + 1 should not work + error
-    try {
-      dfs.setQuota(testFolder, 10, Long.MAX_VALUE + 1);
-      fail("Exception not thrown");
-    } catch (IllegalArgumentException e) {
-      // Expected
+      // setting namespace quota to Long.MAX_VALUE + 1 should not work + error
+      try {
+        dfs.setQuota(testFolder, Long.MAX_VALUE + 1, 10);
+        fail("Exception not thrown");
+      } catch (IllegalArgumentException e) {
+        // Expected
+      }
+    
+      // setting diskspace quota to Long.MAX_VALUE + 1 should not work + error
+      try {
+        dfs.setQuota(testFolder, 10, Long.MAX_VALUE + 1);
+        fail("Exception not thrown");
+      } catch (IllegalArgumentException e) {
+        // Expected
+      }
+    } finally {
+      cluster.shutdown();
     }
   }
   

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java
(original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java
Wed Apr 24 02:21:58 2013
@@ -255,7 +255,6 @@ public class TestEditLogJournalFailures 
       doThrow(new IOException("fail on setReadyToFlush()")).when(spyElos)
         .setReadyToFlush();
     }
-    doNothing().when(spyElos).abort();
   }
 
   private EditLogFileOutputStream spyOnStream(JournalAndStream jas) {

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
(original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
Wed Apr 24 02:21:58 2013
@@ -530,7 +530,11 @@ public class TestStartup {
         .manageDataDfsDirs(false)
         .manageNameDfsDirs(false)
         .build();
-    cluster.waitActive();
+    try {
+      cluster.waitActive();
+    } finally {
+      cluster.shutdown();
+    }
 }
 
   /**

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-22-dfs-dir.tgz
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-22-dfs-dir.tgz?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
Binary files - no diff available.



Mime
View raw message