Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E57E8F3CE for ; Tue, 2 Apr 2013 19:16:14 +0000 (UTC) Received: (qmail 11885 invoked by uid 500); 2 Apr 2013 19:16:14 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 11845 invoked by uid 500); 2 Apr 2013 19:16:14 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 11837 invoked by uid 99); 2 Apr 2013 19:16:14 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 02 Apr 2013 19:16:14 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 02 Apr 2013 19:16:12 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 9476E2388847; Tue, 2 Apr 2013 19:15:52 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1463700 - in /hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/ src/test/java/org/apache/hadoop/hdfs/web/ Date: Tue, 02 Apr 2013 19:15:52 -0000 To: hdfs-commits@hadoop.apache.org From: kihwal@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130402191552.9476E2388847@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: kihwal Date: Tue Apr 2 19:15:52 2013 New Revision: 1463700 URL: http://svn.apache.org/r1463700 Log: HDFS-4649. Webhdfs cannot list large directories. Contributed by Daryn Sharp. Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1463700&r1=1463699&r2=1463700&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Apr 2 19:15:52 2013 @@ -90,6 +90,8 @@ Release 0.23.7 - UNRELEASED HDFS-4581. checkDiskError should not be called on network errors (Rohit Kochar via kihwal) + HDFS-4649. Webhdfs cannot list large directories (daryn via kihwal) + Release 0.23.6 - 2013-02-06 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java?rev=1463700&r1=1463699&r2=1463700&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java Tue Apr 2 19:15:52 2013 @@ -622,9 +622,15 @@ public class NamenodeWebHdfsMethods { private static StreamingOutput getListingStream(final NamenodeProtocols np, final String p) throws IOException { - final DirectoryListing first = getDirectoryListing(np, p, + // allows exceptions like FNF or ACE to prevent http response of 200 for + // a failure since we can't (currently) return error responses in the + // middle of a streaming operation + final DirectoryListing firstDirList = getDirectoryListing(np, p, HdfsFileStatus.EMPTY_NAME); + // must save ugi because the streaming object will be executed outside + // the remote user's ugi + final UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); return new StreamingOutput() { @Override public void write(final OutputStream outstream) throws IOException { @@ -632,21 +638,32 @@ public class NamenodeWebHdfsMethods { out.println("{\"" + FileStatus.class.getSimpleName() + "es\":{\"" + FileStatus.class.getSimpleName() + "\":["); - final HdfsFileStatus[] partial = first.getPartialListing(); - if (partial.length > 0) { - out.print(JsonUtil.toJsonString(partial[0], false)); - } - for(int i = 1; i < partial.length; i++) { - out.println(','); - out.print(JsonUtil.toJsonString(partial[i], false)); - } - - for(DirectoryListing curr = first; curr.hasMore(); ) { - curr = getDirectoryListing(np, p, curr.getLastName()); - for(HdfsFileStatus s : curr.getPartialListing()) { - out.println(','); - out.print(JsonUtil.toJsonString(s, false)); - } + try { + // restore remote user's ugi + ugi.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws IOException { + long n = 0; + for (DirectoryListing dirList = firstDirList; ; + dirList = getDirectoryListing(np, p, dirList.getLastName()) + ) { + // send each segment of the directory listing + for (HdfsFileStatus s : dirList.getPartialListing()) { + if (n++ > 0) { + out.println(','); + } + out.print(JsonUtil.toJsonString(s, false)); + } + // stop if last segment + if (!dirList.hasMore()) { + break; + } + } + return null; + } + }); + } catch (InterruptedException e) { + throw new IOException(e); } out.println(); Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java?rev=1463700&r1=1463699&r2=1463700&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java Tue Apr 2 19:15:52 2013 @@ -19,16 +19,22 @@ package org.apache.hadoop.hdfs.web; import java.io.IOException; +import java.net.URISyntaxException; +import java.security.PrivilegedExceptionAction; import java.util.Random; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.security.UserGroupInformation; import org.junit.Assert; import org.junit.Test; @@ -196,4 +202,48 @@ public class TestWebHDFS { in.close(); t.end(checked); } + + @Test(timeout=300000) + public void testLargeDirectory() throws Exception { + final Configuration conf = WebHdfsTestUtil.createConf(); + final int listLimit = 2; + // force small chunking of directory listing + conf.setInt(DFSConfigKeys.DFS_LIST_LIMIT, listLimit); + // force paths to be only owner-accessible to ensure ugi isn't changing + // during listStatus + FsPermission.setUMask(conf, new FsPermission((short)0077)); + + final MiniDFSCluster cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); + try { + cluster.waitActive(); + WebHdfsTestUtil.getWebHdfsFileSystem(conf).setPermission( + new Path("/"), + new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); + + // trick the NN into not believing it's not the superuser so we can + // tell if the correct user is used by listStatus + UserGroupInformation.setLoginUser( + UserGroupInformation.createUserForTesting( + "not-superuser", new String[]{"not-supergroup"})); + + UserGroupInformation.createUserForTesting("me", new String[]{"my-group"}) + .doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws IOException, URISyntaxException { + FileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf); + Path d = new Path("/my-dir"); + Assert.assertTrue(fs.mkdirs(d)); + for (int i=0; i < listLimit*3; i++) { + Path p = new Path(d, "file-"+i); + Assert.assertTrue(fs.createNewFile(p)); + } + Assert.assertEquals(listLimit*3, fs.listStatus(d).length); + return null; + } + }); + } finally { + cluster.shutdown(); + } + } }