From hdfs-commits-return-847-apmail-hadoop-hdfs-commits-archive=hadoop.apache.org@hadoop.apache.org Wed Sep 22 23:16:03 2010 Return-Path: Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: (qmail 99909 invoked from network); 22 Sep 2010 23:16:03 -0000 Received: from unknown (HELO mail.apache.org) (140.211.11.3) by 140.211.11.9 with SMTP; 22 Sep 2010 23:16:03 -0000 Received: (qmail 95854 invoked by uid 500); 22 Sep 2010 23:16:03 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 95794 invoked by uid 500); 22 Sep 2010 23:16:02 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 95786 invoked by uid 99); 22 Sep 2010 23:16:02 -0000 Received: from Unknown (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 22 Sep 2010 23:16:02 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 22 Sep 2010 23:15:43 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id A2B6123888E8; Wed, 22 Sep 2010 23:15:22 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1000272 - in /hadoop/hdfs/trunk: CHANGES.txt src/test/aop/org/apache/hadoop/hdfs/TestFiHftp.java src/test/aop/org/apache/hadoop/hdfs/server/namenode/FileDataServletAspects.aj Date: Wed, 22 Sep 2010 23:15:22 -0000 To: hdfs-commits@hadoop.apache.org From: szetszwo@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20100922231522.A2B6123888E8@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: szetszwo Date: Wed Sep 22 23:15:22 2010 New Revision: 1000272 URL: http://svn.apache.org/viewvc?rev=1000272&view=rev Log: HDFS-1304. Add a new unit test for HftpFileSystem.open(..). Added: hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/TestFiHftp.java hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/namenode/FileDataServletAspects.aj Modified: hadoop/hdfs/trunk/CHANGES.txt Modified: hadoop/hdfs/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1000272&r1=1000271&r2=1000272&view=diff ============================================================================== --- hadoop/hdfs/trunk/CHANGES.txt (original) +++ hadoop/hdfs/trunk/CHANGES.txt Wed Sep 22 23:15:22 2010 @@ -37,6 +37,8 @@ Trunk (unreleased changes) IMPROVEMENTS + HDFS-1304. Add a new unit test for HftpFileSystem.open(..). (szetszwo) + HDFS-1096. fix for prev. commit. (boryas) HDFS-1096. allow dfsadmin/mradmin refresh of superuser proxy group Added: hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/TestFiHftp.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/TestFiHftp.java?rev=1000272&view=auto ============================================================================== --- hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/TestFiHftp.java (added) +++ hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/TestFiHftp.java Wed Sep 22 23:15:22 2010 @@ -0,0 +1,187 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import java.io.File; +import java.io.IOException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.impl.Log4JLogger; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.ContentSummary; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.protocol.LocatedBlocks; +import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.hdfs.server.datanode.FSDataset; +import org.apache.hadoop.io.IOUtils; +import org.apache.log4j.Level; +import org.junit.Assert; +import org.junit.Test; + +public class TestFiHftp { + final Log LOG = FileSystem.LOG; + { + ((Log4JLogger)LOG).getLogger().setLevel(Level.ALL); + } + + static final short DATANODE_NUM = 1; + static final Random ran = new Random(); + static final byte[] buffer = new byte[1 << 16]; + static final MessageDigest md5; + static { + try { + md5 = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + + private static byte[] createFile(FileSystem fs, Path name, long length, + short replication, long blocksize) throws IOException { + final FSDataOutputStream out = fs.create(name, false, 4096, + replication, blocksize); + try { + for(long n = length; n > 0; ) { + ran.nextBytes(buffer); + final int w = n < buffer.length? (int)n: buffer.length; + out.write(buffer, 0, w); + md5.update(buffer, 0, w); + n -= w; + } + } finally { + IOUtils.closeStream(out); + } + return md5.digest(); + } + + @Test + public void testHftpOpen() throws IOException { + final Configuration conf = new Configuration(); + MiniDFSCluster cluster = null; + try { + cluster = new MiniDFSCluster(conf, DATANODE_NUM, true, null); + cluster.waitActive(); + + //test with a file + //which is larger than the servlet response buffer size + { + final long blocksize = 1L << 20; // + final long filesize = 2*blocksize + 100; + runTestHftpOpen(cluster, "/foo", blocksize, filesize); + } + + //test with a small file + //which is smaller than the servlet response buffer size + { + final long blocksize = 1L << 10; // + final long filesize = 2*blocksize + 100; + runTestHftpOpen(cluster, "/small", blocksize, filesize); + } + } finally { + if (cluster != null) {cluster.shutdown();} + } + } + + /** + * A test with a 3GB file. + * It may take ~6 minutes. + */ + void largeFileTest(final MiniDFSCluster cluster) throws IOException { + final long blocksize = 128L << 20; + final long filesize = 3L << 30; + runTestHftpOpen(cluster, "/large", blocksize, filesize); + } + + /** + * @param blocksize + * @param filesize must be > block size + */ + private void runTestHftpOpen(final MiniDFSCluster cluster, final String file, + final long blocksize, final long filesize) throws IOException { + //create a file + final DistributedFileSystem dfs = (DistributedFileSystem)cluster.getFileSystem(); + final Path filepath = new Path(file); + final byte[] filemd5 = createFile(dfs, filepath, filesize, DATANODE_NUM, + blocksize); + DFSTestUtil.waitReplication(dfs, filepath, DATANODE_NUM); + + //test hftp open and read + final HftpFileSystem hftpfs = cluster.getHftpFileSystem(); + { + final FSDataInputStream in = hftpfs.open(filepath); + long bytesRead = 0; + try { + for(int r; (r = in.read(buffer)) != -1; ) { + bytesRead += r; + md5.update(buffer, 0, r); + } + } finally { + LOG.info("bytesRead=" + bytesRead); + in.close(); + } + Assert.assertEquals(filesize, bytesRead); + Assert.assertArrayEquals(filemd5, md5.digest()); + } + + //delete the second block + final DFSClient client = dfs.getClient(); + final LocatedBlocks locatedblocks = client.getNamenode().getBlockLocations( + file, 0, filesize); + Assert.assertEquals((filesize - 1)/blocksize + 1, + locatedblocks.locatedBlockCount()); + final LocatedBlock lb = locatedblocks.get(1); + final Block blk = lb.getBlock(); + Assert.assertEquals(blocksize, lb.getBlockSize()); + final DatanodeInfo[] datanodeinfos = lb.getLocations(); + Assert.assertEquals(DATANODE_NUM, datanodeinfos.length); + final DataNode dn = cluster.getDataNode(datanodeinfos[0].getIpcPort()); + LOG.info("dn=" + dn + ", blk=" + blk + " (length=" + blk.getNumBytes() + ")"); + final FSDataset data = (FSDataset)dn.getFSDataset(); + final File blkfile = data.getBlockFile(blk); + Assert.assertTrue(blkfile.delete()); + + //read again by hftp, should get an exception + LOG.info("hftpfs.getUri() = " + hftpfs.getUri()); + final ContentSummary cs = hftpfs.getContentSummary(filepath); + LOG.info("hftpfs.getContentSummary = " + cs); + Assert.assertEquals(filesize, cs.getLength()); + + final FSDataInputStream in = hftpfs.open(hftpfs.makeQualified(filepath)); + long bytesRead = 0; + try { + for(int r; (r = in.read(buffer)) != -1; ) { + bytesRead += r; + } + Assert.fail(); + } catch(IOException ioe) { + LOG.info("GOOD: get an exception", ioe); + } finally { + LOG.info("bytesRead=" + bytesRead); + in.close(); + } + } +} Added: hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/namenode/FileDataServletAspects.aj URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/namenode/FileDataServletAspects.aj?rev=1000272&view=auto ============================================================================== --- hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/namenode/FileDataServletAspects.aj (added) +++ hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/namenode/FileDataServletAspects.aj Wed Sep 22 23:15:22 2010 @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import java.net.URI; +import java.net.URISyntaxException; + +import javax.servlet.http.HttpServletRequest; + +import org.apache.commons.logging.Log; +import org.apache.hadoop.hdfs.protocol.ClientProtocol; +import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.security.UserGroupInformation; + +public aspect FileDataServletAspects { + static final Log LOG = FileDataServlet.LOG; + + pointcut callCreateUri() : call (URI FileDataServlet.createUri( + String, HdfsFileStatus, UserGroupInformation, ClientProtocol, + HttpServletRequest, String)); + + /** Replace host name with "localhost" for unit test environment. */ + URI around () throws URISyntaxException : callCreateUri() { + final URI original = proceed(); + LOG.info("FI: original uri = " + original); + final URI replaced = new URI(original.getScheme(), original.getUserInfo(), + "localhost", original.getPort(), original.getPath(), + original.getQuery(), original.getFragment()) ; + LOG.info("FI: replaced uri = " + replaced); + return replaced; + } +}