hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject svn commit: r1354832 [5/5] - in /hadoop/common/branches/HDFS-3092/hadoop-hdfs-project: ./ hadoop-hdfs-httpfs/ hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/ hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/ hadoop-h...
Date Thu, 28 Jun 2012 07:00:25 GMT
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java
(original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java
Thu Jun 28 06:59:38 2012
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.na
 import static org.junit.Assert.*;
 
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URI;
 import java.util.Collections;
@@ -35,6 +36,7 @@ import org.apache.hadoop.hdfs.MiniDFSNNT
 import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
 import org.apache.hadoop.hdfs.server.namenode.NNStorage;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Test;
 
@@ -118,8 +120,8 @@ public class TestEditLogsDuringFailover 
     }
   }
   
-  @Test
-  public void testFailoverFinalizesAndReadsInProgress() throws Exception {
+  private void testFailoverFinalizesAndReadsInProgress(
+      boolean partialTxAtEnd) throws Exception {
     Configuration conf = new Configuration();
     MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
       .nnTopology(MiniDFSNNTopology.simpleHATopology())
@@ -130,8 +132,21 @@ public class TestEditLogsDuringFailover 
       URI sharedUri = cluster.getSharedEditsDir(0, 1);
       File sharedDir = new File(sharedUri.getPath(), "current");
       FSImageTestUtil.createAbortedLogWithMkdirs(sharedDir, NUM_DIRS_IN_LOG, 1);
+      
       assertEditFiles(Collections.singletonList(sharedUri),
           NNStorage.getInProgressEditsFileName(1));
+      if (partialTxAtEnd) {
+        FileOutputStream outs = null;
+        try {
+          File editLogFile =
+              new File(sharedDir, NNStorage.getInProgressEditsFileName(1));
+          outs = new FileOutputStream(editLogFile, true);
+          outs.write(new byte[] { 0x18, 0x00, 0x00, 0x00 } );
+          LOG.error("editLogFile = " + editLogFile);
+        } finally {
+          IOUtils.cleanup(LOG, outs);
+        }
+     }
 
       // Transition one of the NNs to active
       cluster.transitionToActive(0);
@@ -149,7 +164,18 @@ public class TestEditLogsDuringFailover 
     } finally {
       cluster.shutdown();
     }
+  }
+  
+  @Test
+  public void testFailoverFinalizesAndReadsInProgressSimple()
+      throws Exception {
+    testFailoverFinalizesAndReadsInProgress(false);
+  }
 
+  @Test
+  public void testFailoverFinalizesAndReadsInProgressWithPartialTxAtEnd()
+      throws Exception {
+    testFailoverFinalizesAndReadsInProgress(true);
   }
 
   /**

Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java
(original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java
Thu Jun 28 06:59:38 2012
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hdfs.server.namenode.ha;
 
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY;
 import static org.junit.Assert.*;
 
 import java.io.File;
@@ -127,6 +129,7 @@ public class TestFailureOfSharedDir {
   @Test
   public void testFailureOfSharedDir() throws Exception {
     Configuration conf = new Configuration();
+    conf.setLong(DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY, 2000);
     
     // The shared edits dir will automatically be marked required.
     MiniDFSCluster cluster = null;
@@ -151,6 +154,15 @@ public class TestFailureOfSharedDir {
       assertEquals(0, FileUtil.chmod(sharedEditsDir.getAbsolutePath(), "-w",
           true));
 
+      Thread.sleep(conf.getLong(DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY,
+          DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_DEFAULT) * 2);
+
+      NameNode nn1 = cluster.getNameNode(1);
+      assertTrue(nn1.isStandbyState());
+      assertFalse(
+          "StandBy NameNode should not go to SafeMode on resource unavailability",
+          nn1.isInSafeMode());
+
       NameNode nn0 = cluster.getNameNode(0);
       nn0.getNamesystem().getFSImage().getEditLog().getJournalSet()
           .setRuntimeForTesting(mockRuntime);

Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
(original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
Thu Jun 28 06:59:38 2012
@@ -22,11 +22,14 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.File;
 import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
 import java.util.Map;
 import java.util.HashMap;
 
 import org.junit.Test;
 import org.junit.Before;
+
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 import org.apache.commons.logging.Log;
@@ -34,12 +37,12 @@ import org.apache.commons.logging.LogFac
 
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes;
 import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer;
+import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer.Flags;
 import org.apache.hadoop.hdfs.DFSTestUtil;
 
 import org.apache.hadoop.hdfs.server.namenode.OfflineEditsViewerHelper;
 
 public class TestOfflineEditsViewer {
-
   private static final Log LOG = LogFactory.getLog(TestOfflineEditsViewer.class);
 
   private static final Map<FSEditLogOpCodes, Boolean> obsoleteOpCodes =
@@ -97,8 +100,8 @@ public class TestOfflineEditsViewer {
     String editsReparsed  = cacheDir + "/editsReparsed";
 
     // parse to XML then back to binary
-    runOev(edits,          editsParsedXml, "xml");
-    runOev(editsParsedXml, editsReparsed,  "binary");
+    assertEquals(0, runOev(edits, editsParsedXml, "xml", false));
+    assertEquals(0, runOev(editsParsedXml, editsReparsed, "binary", false));
 
     // judgment time
     assertTrue(
@@ -115,6 +118,42 @@ public class TestOfflineEditsViewer {
   }
 
   @Test
+  public void testRecoveryMode() throws IOException {
+    LOG.info("START - testing with generated edits");
+
+    nnHelper.startCluster(buildDir + "/dfs/");
+
+    // edits generated by nnHelper (MiniDFSCluster), should have all op codes
+    // binary, XML, reparsed binary
+    String edits          = nnHelper.generateEdits();
+    
+    // Corrupt the file by truncating the end
+    FileChannel editsFile = new FileOutputStream(edits, true).getChannel();
+    editsFile.truncate(editsFile.size() - 5);
+    
+    String editsParsedXml = cacheDir + "/editsRecoveredParsed.xml";
+    String editsReparsed  = cacheDir + "/editsRecoveredReparsed";
+    String editsParsedXml2 = cacheDir + "/editsRecoveredParsed2.xml";
+
+    // Can't read the corrupted file without recovery mode
+    assertEquals(-1, runOev(edits, editsParsedXml, "xml", false));
+    
+    // parse to XML then back to binary
+    assertEquals(0, runOev(edits, editsParsedXml, "xml", true));
+    assertEquals(0, runOev(editsParsedXml, editsReparsed,  "binary", false));
+    assertEquals(0, runOev(editsReparsed, editsParsedXml2, "xml", false));
+
+    // judgment time
+    assertTrue("Test round trip",
+      filesEqualIgnoreTrailingZeros(editsParsedXml, editsParsedXml2));
+
+    // removes edits so do this at the end
+    nnHelper.shutdownCluster();
+
+    LOG.info("END");
+  }
+
+  @Test
   public void testStored() throws IOException {
 
     LOG.info("START - testing with stored reference edits");
@@ -128,8 +167,9 @@ public class TestOfflineEditsViewer {
     String editsStoredXml          = cacheDir + "/editsStored.xml";
       
     // parse to XML then back to binary
-    runOev(editsStored,             editsStoredParsedXml, "xml");
-    runOev(editsStoredParsedXml,    editsStoredReparsed,  "binary");
+    assertEquals(0, runOev(editsStored, editsStoredParsedXml, "xml", false));
+    assertEquals(0, runOev(editsStoredParsedXml, editsStoredReparsed,
+        "binary", false));
 
     // judgement time
     assertTrue(
@@ -151,14 +191,18 @@ public class TestOfflineEditsViewer {
    * @param inFilename input edits filename
    * @param outFilename oputput edits filename
    */
-  private void runOev(String inFilename, String outFilename, String processor)
-    throws IOException {
+  private int runOev(String inFilename, String outFilename, String processor,
+      boolean recovery) throws IOException {
 
     LOG.info("Running oev [" + inFilename + "] [" + outFilename + "]");
 
     OfflineEditsViewer oev = new OfflineEditsViewer();
-    if (oev.go(inFilename, outFilename, processor, true, false, null) != 0)
-      throw new RuntimeException("oev failed");
+    Flags flags = new Flags();
+    flags.setPrintToScreen();
+    if (recovery) {
+      flags.setRecoveryMode();
+    }
+    return oev.go(inFilename, outFilename, processor, flags, null);
   }
 
   /**
@@ -172,7 +216,7 @@ public class TestOfflineEditsViewer {
     FileOutputStream fout = new FileOutputStream(outFilename);
     StatisticsEditsVisitor visitor = new StatisticsEditsVisitor(fout);
     OfflineEditsViewer oev = new OfflineEditsViewer();
-    if (oev.go(inFilename, outFilename, "stats", false, false, visitor) != 0)
+    if (oev.go(inFilename, outFilename, "stats", new Flags(), visitor) != 0)
       return false;
     LOG.info("Statistics for " + inFilename + "\n" +
       visitor.getStatisticsString());

Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
(original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
Thu Jun 28 06:59:38 2012
@@ -44,6 +44,7 @@ import org.apache.hadoop.hdfs.MiniDFSClu
 import org.apache.hadoop.hdfs.web.resources.DoAsParam;
 import org.apache.hadoop.hdfs.web.resources.GetOpParam;
 import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
+import org.apache.hadoop.hdfs.web.resources.NamenodeRpcAddressParam;
 import org.apache.hadoop.hdfs.web.resources.PutOpParam;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -286,6 +287,10 @@ public class TestWebHdfsFileSystemContra
     final Path root = new Path("/");
     final Path dir = new Path("/test/testUrl");
     assertTrue(webhdfs.mkdirs(dir));
+    final Path file = new Path("/test/file");
+    final FSDataOutputStream out = webhdfs.create(file);
+    out.write(1);
+    out.close();
 
     {//test GETHOMEDIRECTORY
       final URL url = webhdfs.toUrl(GetOpParam.Op.GETHOMEDIRECTORY, root);
@@ -351,5 +356,47 @@ public class TestWebHdfsFileSystemContra
     {//test append.
       AppendTestUtil.testAppend(fs, new Path(dir, "append"));
     }
+
+    {//test NamenodeRpcAddressParam not set.
+      final HttpOpParam.Op op = PutOpParam.Op.CREATE;
+      final URL url = webhdfs.toUrl(op, dir);
+      HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+      conn.setRequestMethod(op.getType().toString());
+      conn.setDoOutput(false);
+      conn.setInstanceFollowRedirects(false);
+      conn.connect();
+      final String redirect = conn.getHeaderField("Location");
+      conn.disconnect();
+
+      //remove NamenodeRpcAddressParam
+      WebHdfsFileSystem.LOG.info("redirect = " + redirect);
+      final int i = redirect.indexOf(NamenodeRpcAddressParam.NAME);
+      final int j = redirect.indexOf("&", i);
+      String modified = redirect.substring(0, i - 1) + redirect.substring(j);
+      WebHdfsFileSystem.LOG.info("modified = " + modified);
+
+      //connect to datanode
+      conn = (HttpURLConnection)new URL(modified).openConnection();
+      conn.setRequestMethod(op.getType().toString());
+      conn.setDoOutput(op.getDoOutput());
+      conn.connect();
+      assertEquals(HttpServletResponse.SC_BAD_REQUEST, conn.getResponseCode());
+    }
+
+    {//test jsonParse with non-json type.
+      final HttpOpParam.Op op = GetOpParam.Op.OPEN;
+      final URL url = webhdfs.toUrl(op, file);
+      final HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+      conn.setRequestMethod(op.getType().toString());
+      conn.connect();
+
+      try {
+        WebHdfsFileSystem.jsonParse(conn, false);
+        fail();
+      } catch(IOException ioe) {
+        WebHdfsFileSystem.LOG.info("GOOD", ioe);
+      }
+      conn.disconnect();
+    }
   }
 }

Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java
(original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java
Thu Jun 28 06:59:38 2012
@@ -40,6 +40,12 @@ import org.junit.Assert;
 public class WebHdfsTestUtil {
   public static final Log LOG = LogFactory.getLog(WebHdfsTestUtil.class);
 
+  public static Configuration createConf() {
+    final Configuration conf = new Configuration();
+    conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true);
+    return conf;
+  }
+
   public static WebHdfsFileSystem getWebHdfsFileSystem(final Configuration conf
       ) throws IOException, URISyntaxException {
     final String uri = WebHdfsFileSystem.SCHEME  + "://"
@@ -49,7 +55,7 @@ public class WebHdfsTestUtil {
 
   public static WebHdfsFileSystem getWebHdfsFileSystemAs(
       final UserGroupInformation ugi, final Configuration conf
-      ) throws IOException, URISyntaxException, InterruptedException {
+      ) throws IOException, InterruptedException {
     return ugi.doAs(new PrivilegedExceptionAction<WebHdfsFileSystem>() {
       @Override
       public WebHdfsFileSystem run() throws Exception {
@@ -70,7 +76,7 @@ public class WebHdfsTestUtil {
       final int expectedResponseCode) throws IOException {
     conn.connect();
     Assert.assertEquals(expectedResponseCode, conn.getResponseCode());
-    return WebHdfsFileSystem.jsonParse(conn.getInputStream());
+    return WebHdfsFileSystem.jsonParse(conn, false);
   }
   
   public static HttpURLConnection twoStepWrite(HttpURLConnection conn,

Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
(original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
Thu Jun 28 06:59:38 2012
@@ -15470,7 +15470,7 @@
       <comparators>
         <comparator>
           <type>SubstringComparator</type>
-          <expected-output>For input string: "a5"</expected-output>
+          <expected-output>setSpaceQuota: "a5" is not a valid value for a quota.</expected-output>
         </comparator>
       </comparators>
     </test>

Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/pom.xml?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/pom.xml (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/pom.xml Thu Jun 28 06:59:38 2012
@@ -34,7 +34,7 @@ http://maven.apache.org/xsd/maven-4.0.0.
     <module>hadoop-hdfs</module>
     <module>hadoop-hdfs-httpfs</module>
     <module>hadoop-hdfs/src/contrib/bkjournal</module>
-    <module>hadoop-hdfs/src/contrib/fuse-dfs</module>
+    <module>hadoop-hdfs-raid</module>
   </modules>
 
   <build>



Mime
View raw message