hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1143523 [4/4] - in /hadoop/common/branches/HDFS-1073/hdfs: ./ src/c++/libhdfs/ src/contrib/hdfsproxy/ src/java/ src/java/org/apache/hadoop/hdfs/ src/java/org/apache/hadoop/hdfs/protocol/ src/java/org/apache/hadoop/hdfs/protocol/datatransfe...
Date Wed, 06 Jul 2011 18:32:07 GMT
Modified: hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStreamFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStreamFile.java?rev=1143523&r1=1143522&r2=1143523&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStreamFile.java
(original)
+++ hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStreamFile.java
Wed Jul  6 18:32:04 2011
@@ -20,17 +20,30 @@ package org.apache.hadoop.hdfs.server.na
 import static org.junit.Assert.assertArrayEquals;
 
 import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Enumeration;
 import java.util.List;
 import java.util.Vector;
 
+import javax.servlet.ServletContext;
+import javax.servlet.ServletOutputStream;
+import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
 
 import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSInputStream;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.common.JspHelper;
+import org.mockito.Mockito;
 import org.mortbay.jetty.InclusiveByteRange;
 
 /*
@@ -186,9 +199,30 @@ class MockHttpServletResponse implements
 }
 
 
+public class TestStreamFile {
+  private HdfsConfiguration CONF = new HdfsConfiguration();
+  private DFSClient clientMock = Mockito.mock(DFSClient.class);
+  private HttpServletRequest mockHttpServletRequest = 
+    Mockito.mock(HttpServletRequest.class);
+  private HttpServletResponse mockHttpServletResponse = 
+    Mockito.mock(HttpServletResponse.class);
+  private final ServletContext mockServletContext = 
+    Mockito.mock(ServletContext.class);
 
-public class TestStreamFile extends TestCase {
+  StreamFile sfile = new StreamFile() {
+    private static final long serialVersionUID = -5513776238875189473L;
   
+    public ServletContext getServletContext() {
+      return mockServletContext;
+    }
+  
+    @Override
+    protected DFSClient getDFSClient(HttpServletRequest request)
+      throws IOException, InterruptedException {
+      return clientMock;
+    }
+  };
+     
   // return an array matching the output of mockfsinputstream
   private static byte[] getOutputArray(int start, int count) {
     byte[] a = new byte[count];
@@ -200,6 +234,7 @@ public class TestStreamFile extends Test
     return a;
   }
   
+  @Test
   public void testWriteTo() throws IOException, InterruptedException {
 
     FSInputStream fsin = new MockFSInputStream();
@@ -219,7 +254,7 @@ public class TestStreamFile extends Test
     assertTrue("Pairs array must be even", pairs.length % 2 == 0);
     
     for (int i = 0; i < pairs.length; i+=2) {
-      StreamFile.writeTo(fsin, os, pairs[i], pairs[i+1]);
+      StreamFile.copyFromOffset(fsin, os, pairs[i], pairs[i+1]);
       assertArrayEquals("Reading " + pairs[i+1]
                         + " bytes from offset " + pairs[i],
                         getOutputArray(pairs[i], pairs[i+1]),
@@ -228,20 +263,22 @@ public class TestStreamFile extends Test
     }
     
   }
-  
-  private List<?> strToRanges(String s, int contentLength) {
+
+  @SuppressWarnings("unchecked")
+  private List<InclusiveByteRange> strToRanges(String s, int contentLength) {
     List<String> l = Arrays.asList(new String[]{"bytes="+s});
     Enumeration<?> e = (new Vector<String>(l)).elements();
     return InclusiveByteRange.satisfiableRanges(e, contentLength);
   }
   
+  @Test
   public void testSendPartialData() throws IOException, InterruptedException {
     FSInputStream in = new MockFSInputStream();
     ByteArrayOutputStream os = new ByteArrayOutputStream();
 
     // test if multiple ranges, then 416
     { 
-      List<?> ranges = strToRanges("0-,10-300", 500);
+      List<InclusiveByteRange> ranges = strToRanges("0-,10-300", 500);
       MockHttpServletResponse response = new MockHttpServletResponse();
       StreamFile.sendPartialData(in, os, response, 500, ranges);
       assertEquals("Multiple ranges should result in a 416 error",
@@ -259,7 +296,7 @@ public class TestStreamFile extends Test
 
     // test if invalid single range (out of bounds), then 416
     { 
-      List<?> ranges = strToRanges("600-800", 500);
+      List<InclusiveByteRange> ranges = strToRanges("600-800", 500);
       MockHttpServletResponse response = new MockHttpServletResponse();
       StreamFile.sendPartialData(in, os, response, 500, ranges);
       assertEquals("Single (but invalid) range should result in a 416",
@@ -269,7 +306,7 @@ public class TestStreamFile extends Test
       
     // test if one (valid) range, then 206
     { 
-      List<?> ranges = strToRanges("100-300", 500);
+      List<InclusiveByteRange> ranges = strToRanges("100-300", 500);
       MockHttpServletResponse response = new MockHttpServletResponse();
       StreamFile.sendPartialData(in, os, response, 500, ranges);
       assertEquals("Single (valid) range should result in a 206",
@@ -280,4 +317,108 @@ public class TestStreamFile extends Test
     }
     
   }
+  
+  
+    // Test for positive scenario
+  @Test
+  public void testDoGetShouldWriteTheFileContentIntoServletOutputStream()
+      throws Exception {
+
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(1)
+        .build();
+    try {
+      Path testFile = createFile();
+      setUpForDoGetTest(cluster, testFile);
+      ServletOutputStreamExtn outStream = new ServletOutputStreamExtn();
+      Mockito.doReturn(outStream).when(mockHttpServletResponse)
+          .getOutputStream();
+      StreamFile sfile = new StreamFile() {
+
+        private static final long serialVersionUID = 7715590481809562722L;
+
+        public ServletContext getServletContext() {
+          return mockServletContext;
+        }
+      };
+      sfile.doGet(mockHttpServletRequest, mockHttpServletResponse);
+      assertEquals("Not writing the file data into ServletOutputStream",
+          outStream.getResult(), "test");
+    } finally {
+      cluster.shutdown();
+    }
+  }
+
+  // Test for cleaning the streams in exception cases also
+  @Test
+  public void testDoGetShouldCloseTheDFSInputStreamIfResponseGetOutPutStreamThrowsAnyException()
+      throws Exception {
+
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(1)
+        .build();
+    try {
+      Path testFile = createFile();
+
+      setUpForDoGetTest(cluster, testFile);
+
+      Mockito.doThrow(new IOException()).when(mockHttpServletResponse)
+          .getOutputStream();
+      DFSInputStream fsMock = Mockito.mock(DFSInputStream.class);
+
+      Mockito.doReturn(fsMock).when(clientMock).open(testFile.toString());
+
+      Mockito.doReturn(Long.valueOf(4)).when(fsMock).getFileLength();
+
+      try {
+        sfile.doGet(mockHttpServletRequest, mockHttpServletResponse);
+        fail("Not throwing the IOException");
+      } catch (IOException e) {
+        Mockito.verify(clientMock, Mockito.atLeastOnce()).close();
+      }
+
+    } finally {
+      cluster.shutdown();
+    }
+  }
+
+  private void setUpForDoGetTest(MiniDFSCluster cluster, Path testFile)
+      throws IOException {
+
+    Mockito.doReturn(CONF).when(mockServletContext).getAttribute(
+        JspHelper.CURRENT_CONF);
+    Mockito.doReturn(NameNode.getHostPortString(NameNode.getAddress(CONF)))
+        .when(mockHttpServletRequest).getParameter("nnaddr");
+    Mockito.doReturn(testFile.toString()).when(mockHttpServletRequest)
+        .getPathInfo();
+  }
+
+  static Path writeFile(FileSystem fs, Path f) throws IOException {
+    DataOutputStream out = fs.create(f);
+    try {
+      out.writeBytes("test");
+    } finally {
+      out.close();
+    }
+    assertTrue(fs.exists(f));
+    return f;
+  }
+
+  private Path createFile() throws IOException {
+    FileSystem fs = FileSystem.get(CONF);
+    Path testFile = new Path("/test/mkdirs/doGet");
+    writeFile(fs, testFile);
+    return testFile;
+  }
+
+  public static class ServletOutputStreamExtn extends ServletOutputStream {
+    private StringBuffer buffer = new StringBuffer(3);
+
+    public String getResult() {
+      return buffer.toString();
+    }
+
+    @Override
+    public void write(int b) throws IOException {
+      buffer.append((char) b);
+    }
+  }
 }

Modified: hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java?rev=1143523&r1=1143522&r2=1143523&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
(original)
+++ hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
Wed Jul  6 18:32:04 2011
@@ -17,34 +17,35 @@
  */
 package org.apache.hadoop.hdfs.server.namenode.metrics;
 
+import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
+import static org.apache.hadoop.test.MetricsAsserts.assertGauge;
+import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+
 import java.io.DataInputStream;
 import java.io.IOException;
 import java.util.Random;
 
 import junit.framework.TestCase;
 
+import org.apache.commons.logging.LogFactory;
 import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.Options.Rename;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.hdfs.server.namenode.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.metrics2.MetricsRecordBuilder;
 import org.apache.hadoop.test.MetricsAsserts;
 import org.apache.log4j.Level;
 
-import org.apache.commons.logging.LogFactory;
-
-import static org.apache.hadoop.test.MetricsAsserts.*;
-
 /**
  * Test for metrics published by the Namenode
  */

Modified: hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/net/TestNetworkTopology.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/net/TestNetworkTopology.java?rev=1143523&r1=1143522&r2=1143523&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/net/TestNetworkTopology.java
(original)
+++ hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/net/TestNetworkTopology.java
Wed Jul  6 18:32:04 2011
@@ -24,8 +24,8 @@ import java.util.Map;
 
 import junit.framework.TestCase;
 
-import org.apache.hadoop.hdfs.server.namenode.DatanodeDescriptor;
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
 
 public class TestNetworkTopology extends TestCase {
   private final static NetworkTopology cluster = new NetworkTopology();

Modified: hadoop/common/branches/HDFS-1073/hdfs/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java?rev=1143523&r1=1143522&r2=1143523&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1073/hdfs/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java
(original)
+++ hadoop/common/branches/HDFS-1073/hdfs/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java
Wed Jul  6 18:32:04 2011
@@ -19,6 +19,16 @@
 package org.apache.hadoop.hdfs.server.namenode;
 
 import static junit.framework.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+import java.io.File;
+import java.io.IOException;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.commons.logging.impl.Log4JLogger;
@@ -34,20 +44,16 @@ import org.apache.hadoop.hdfs.HdfsConfig
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
 import org.apache.hadoop.hdfs.server.common.HdfsConstants;
 import org.apache.hadoop.hdfs.server.common.HdfsConstants.NamenodeRole;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.log4j.Level;
 import org.junit.After;
-
-import static org.junit.Assert.*;
 import org.junit.Before;
 import org.junit.Test;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.*;
-
-import java.io.File;
-import java.io.IOException;
 
 public class TestNNLeaseRecovery {
   private static final Log LOG = LogFactory.getLog(TestNNLeaseRecovery.class);

Propchange: hadoop/common/branches/HDFS-1073/hdfs/src/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul  6 18:32:04 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hdfs/src/webapps/datanode:1134994-1138149
+/hadoop/common/trunk/hdfs/src/webapps/datanode:1134994-1143516
 /hadoop/core/branches/branch-0.19/hdfs/src/webapps/datanode:713112
 /hadoop/core/trunk/src/webapps/datanode:776175-784663
 /hadoop/hdfs/branches/HDFS-1052/src/webapps/datanode:987665-1095512

Propchange: hadoop/common/branches/HDFS-1073/hdfs/src/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul  6 18:32:04 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hdfs/src/webapps/hdfs:1134994-1138149
+/hadoop/common/trunk/hdfs/src/webapps/hdfs:1134994-1143516
 /hadoop/core/branches/branch-0.19/hdfs/src/webapps/hdfs:713112
 /hadoop/core/trunk/src/webapps/hdfs:776175-784663
 /hadoop/hdfs/branches/HDFS-1052/src/webapps/hdfs:987665-1095512

Modified: hadoop/common/branches/HDFS-1073/hdfs/src/webapps/hdfs/block_info_xml.jsp
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/webapps/hdfs/block_info_xml.jsp?rev=1143523&r1=1143522&r2=1143523&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1073/hdfs/src/webapps/hdfs/block_info_xml.jsp (original)
+++ hadoop/common/branches/HDFS-1073/hdfs/src/webapps/hdfs/block_info_xml.jsp Wed Jul  6 18:32:04
2011
@@ -59,19 +59,9 @@
 %>
 <%@ page
   contentType="application/xml"
-  import="java.io.IOException"
-  import="java.util.Iterator"
-  import="org.apache.hadoop.conf.Configuration"
-  import="org.apache.hadoop.hdfs.protocol.Block"
-  import="org.apache.hadoop.hdfs.server.namenode.INode"
-  import="org.apache.hadoop.hdfs.server.namenode.BlocksMap"
-  import="org.apache.hadoop.hdfs.server.namenode.BlockInfo"
-  import="org.apache.hadoop.hdfs.server.namenode.DatanodeDescriptor"
   import="org.apache.hadoop.hdfs.server.namenode.NamenodeJspHelper.XMLBlockInfo"
   import="org.apache.hadoop.hdfs.server.common.JspHelper"
-  import="org.apache.hadoop.util.ServletUtil"
   import="org.znerd.xmlenc.*"
-  
 %>
 <%!
   //for java.io.Serializable

Propchange: hadoop/common/branches/HDFS-1073/hdfs/src/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul  6 18:32:04 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hdfs/src/webapps/secondary:1134994-1138149
+/hadoop/common/trunk/hdfs/src/webapps/secondary:1134994-1143516
 /hadoop/core/branches/branch-0.19/hdfs/src/webapps/secondary:713112
 /hadoop/core/trunk/src/webapps/secondary:776175-784663
 /hadoop/hdfs/branches/HDFS-1052/src/webapps/secondary:987665-1095512



Mime
View raw message