Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 7A9B411BE7 for ; Tue, 19 Aug 2014 23:52:48 +0000 (UTC) Received: (qmail 49903 invoked by uid 500); 19 Aug 2014 23:52:48 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 49854 invoked by uid 500); 19 Aug 2014 23:52:48 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 49841 invoked by uid 99); 19 Aug 2014 23:52:47 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 19 Aug 2014 23:52:47 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 19 Aug 2014 23:51:10 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 9C7BB2388C9C; Tue, 19 Aug 2014 23:50:48 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1619012 [5/35] - in /hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project: hadoop-hdfs-httpfs/ hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/ hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/ hadoop-... Date: Tue, 19 Aug 2014 23:50:25 -0000 To: hdfs-commits@hadoop.apache.org From: cmccabe@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140819235048.9C7BB2388C9C@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java Tue Aug 19 23:49:39 2014 @@ -17,15 +17,602 @@ */ package org.apache.hadoop.hdfs.nfs.nfs3; -import org.apache.hadoop.nfs.nfs3.Nfs3Constant; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import org.jboss.netty.channel.Channel; +import org.junit.AfterClass; import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; +import org.mockito.Mockito; + +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration; +import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys; +import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.nfs.nfs3.FileHandle; +import org.apache.hadoop.nfs.nfs3.Nfs3Constant; +import org.apache.hadoop.nfs.nfs3.Nfs3Constant.WriteStableHow; +import org.apache.hadoop.nfs.nfs3.Nfs3Status; +import org.apache.hadoop.nfs.nfs3.request.LOOKUP3Request; +import org.apache.hadoop.nfs.nfs3.request.READ3Request; +import org.apache.hadoop.nfs.nfs3.request.WRITE3Request; +import org.apache.hadoop.nfs.nfs3.response.ACCESS3Response; +import org.apache.hadoop.nfs.nfs3.response.COMMIT3Response; +import org.apache.hadoop.nfs.nfs3.response.CREATE3Response; +import org.apache.hadoop.nfs.nfs3.response.FSSTAT3Response; +import org.apache.hadoop.nfs.nfs3.response.FSINFO3Response; +import org.apache.hadoop.nfs.nfs3.response.GETATTR3Response; +import org.apache.hadoop.nfs.nfs3.response.LOOKUP3Response; +import org.apache.hadoop.nfs.nfs3.response.PATHCONF3Response; +import org.apache.hadoop.nfs.nfs3.response.READ3Response; +import org.apache.hadoop.nfs.nfs3.response.REMOVE3Response; +import org.apache.hadoop.nfs.nfs3.response.RMDIR3Response; +import org.apache.hadoop.nfs.nfs3.response.RENAME3Response; +import org.apache.hadoop.nfs.nfs3.response.READDIR3Response; +import org.apache.hadoop.nfs.nfs3.response.READDIRPLUS3Response; +import org.apache.hadoop.nfs.nfs3.response.READLINK3Response; +import org.apache.hadoop.nfs.nfs3.response.SETATTR3Response; +import org.apache.hadoop.nfs.nfs3.response.SYMLINK3Response; +import org.apache.hadoop.nfs.nfs3.response.WRITE3Response; +import org.apache.hadoop.nfs.nfs3.request.SetAttr3; +import org.apache.hadoop.oncrpc.XDR; +import org.apache.hadoop.oncrpc.security.SecurityHandler; +import org.apache.hadoop.security.authorize.DefaultImpersonationProvider; +import org.apache.hadoop.security.authorize.ProxyUsers; /** * Tests for {@link RpcProgramNfs3} */ public class TestRpcProgramNfs3 { + static DistributedFileSystem hdfs; + static MiniDFSCluster cluster = null; + static NfsConfiguration config = new NfsConfiguration(); + static NameNode nn; + static Nfs3 nfs; + static RpcProgramNfs3 nfsd; + static SecurityHandler securityHandler; + static SecurityHandler securityHandlerUnpriviledged; + static String testdir = "/tmp"; + + @BeforeClass + public static void setup() throws Exception { + String currentUser = System.getProperty("user.name"); + + config.set("fs.permissions.umask-mode", "u=rwx,g=,o="); + config.set(DefaultImpersonationProvider.getTestProvider() + .getProxySuperuserGroupConfKey(currentUser), "*"); + config.set(DefaultImpersonationProvider.getTestProvider() + .getProxySuperuserIpConfKey(currentUser), "*"); + ProxyUsers.refreshSuperUserGroupsConfiguration(config); + + cluster = new MiniDFSCluster.Builder(config).numDataNodes(1).build(); + cluster.waitActive(); + hdfs = cluster.getFileSystem(); + nn = cluster.getNameNode(); + + // Use ephemeral ports in case tests are running in parallel + config.setInt("nfs3.mountd.port", 0); + config.setInt("nfs3.server.port", 0); + + // Start NFS with allowed.hosts set to "* rw" + config.set("dfs.nfs.exports.allowed.hosts", "* rw"); + nfs = new Nfs3(config); + nfs.startServiceInternal(false); + nfsd = (RpcProgramNfs3) nfs.getRpcProgram(); + + + // Mock SecurityHandler which returns system user.name + securityHandler = Mockito.mock(SecurityHandler.class); + Mockito.when(securityHandler.getUser()).thenReturn(currentUser); + + // Mock SecurityHandler which returns a dummy username "harry" + securityHandlerUnpriviledged = Mockito.mock(SecurityHandler.class); + Mockito.when(securityHandlerUnpriviledged.getUser()).thenReturn("harry"); + } + + @AfterClass + public static void shutdown() throws Exception { + if (cluster != null) { + cluster.shutdown(); + } + } + + @Before + public void createFiles() throws IllegalArgumentException, IOException { + hdfs.delete(new Path(testdir), true); + hdfs.mkdirs(new Path(testdir)); + hdfs.mkdirs(new Path(testdir + "/foo")); + DFSTestUtil.createFile(hdfs, new Path(testdir + "/bar"), 0, (short) 1, 0); + } + + @Test(timeout = 60000) + public void testGetattr() throws Exception { + HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar"); + long dirId = status.getFileId(); + FileHandle handle = new FileHandle(dirId); + XDR xdr_req = new XDR(); + handle.serialize(xdr_req); + + // Attempt by an unpriviledged user should fail. + GETATTR3Response response1 = nfsd.getattr(xdr_req.asReadOnlyWrap(), + securityHandlerUnpriviledged, + new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code", Nfs3Status.NFS3ERR_ACCES, + response1.getStatus()); + + // Attempt by a priviledged user should pass. + GETATTR3Response response2 = nfsd.getattr(xdr_req.asReadOnlyWrap(), + securityHandler, new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code", Nfs3Status.NFS3_OK, + response2.getStatus()); + } + + @Test(timeout = 60000) + public void testSetattr() throws Exception { + HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir); + long dirId = status.getFileId(); + XDR xdr_req = new XDR(); + FileHandle handle = new FileHandle(dirId); + handle.serialize(xdr_req); + xdr_req.writeString("bar"); + SetAttr3 symAttr = new SetAttr3(); + symAttr.serialize(xdr_req); + xdr_req.writeBoolean(false); + + // Attempt by an unpriviledged user should fail. + SETATTR3Response response1 = nfsd.setattr(xdr_req.asReadOnlyWrap(), + securityHandlerUnpriviledged, + new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code", Nfs3Status.NFS3ERR_ACCES, + response1.getStatus()); + + // Attempt by a priviledged user should pass. + SETATTR3Response response2 = nfsd.setattr(xdr_req.asReadOnlyWrap(), + securityHandler, new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code", Nfs3Status.NFS3_OK, + response2.getStatus()); + } + + @Test(timeout = 60000) + public void testLookup() throws Exception { + HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir); + long dirId = status.getFileId(); + FileHandle handle = new FileHandle(dirId); + LOOKUP3Request lookupReq = new LOOKUP3Request(handle, "bar"); + XDR xdr_req = new XDR(); + lookupReq.serialize(xdr_req); + + // Attempt by an unpriviledged user should fail. + LOOKUP3Response response1 = nfsd.lookup(xdr_req.asReadOnlyWrap(), + securityHandlerUnpriviledged, + new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code", Nfs3Status.NFS3ERR_ACCES, + response1.getStatus()); + + // Attempt by a priviledged user should pass. + LOOKUP3Response response2 = nfsd.lookup(xdr_req.asReadOnlyWrap(), + securityHandler, new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code", Nfs3Status.NFS3_OK, + response2.getStatus()); + } + + @Test(timeout = 60000) + public void testAccess() throws Exception { + HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar"); + long dirId = status.getFileId(); + FileHandle handle = new FileHandle(dirId); + XDR xdr_req = new XDR(); + handle.serialize(xdr_req); + + // Attempt by an unpriviledged user should fail. + ACCESS3Response response1 = nfsd.access(xdr_req.asReadOnlyWrap(), + securityHandlerUnpriviledged, + new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code", Nfs3Status.NFS3ERR_ACCES, + response1.getStatus()); + + // Attempt by a priviledged user should pass. + ACCESS3Response response2 = nfsd.access(xdr_req.asReadOnlyWrap(), + securityHandler, new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code", Nfs3Status.NFS3_OK, + response2.getStatus()); + } + + @Test(timeout = 60000) + public void testReadlink() throws Exception { + // Create a symlink first. + HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir); + long dirId = status.getFileId(); + XDR xdr_req = new XDR(); + FileHandle handle = new FileHandle(dirId); + handle.serialize(xdr_req); + xdr_req.writeString("fubar"); + SetAttr3 symAttr = new SetAttr3(); + symAttr.serialize(xdr_req); + xdr_req.writeString("bar"); + + SYMLINK3Response response = nfsd.symlink(xdr_req.asReadOnlyWrap(), + securityHandler, new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK, + response.getStatus()); + + // Now perform readlink operations. + FileHandle handle2 = response.getObjFileHandle(); + XDR xdr_req2 = new XDR(); + handle2.serialize(xdr_req2); + + // Attempt by an unpriviledged user should fail. + READLINK3Response response1 = nfsd.readlink(xdr_req2.asReadOnlyWrap(), + securityHandlerUnpriviledged, + new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES, + response1.getStatus()); + + // Attempt by a priviledged user should pass. + READLINK3Response response2 = nfsd.readlink(xdr_req2.asReadOnlyWrap(), + securityHandler, new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK, + response2.getStatus()); + } + + @Test(timeout = 60000) + public void testRead() throws Exception { + HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar"); + long dirId = status.getFileId(); + FileHandle handle = new FileHandle(dirId); + + READ3Request readReq = new READ3Request(handle, 0, 5); + XDR xdr_req = new XDR(); + readReq.serialize(xdr_req); + + // Attempt by an unpriviledged user should fail. + READ3Response response1 = nfsd.read(xdr_req.asReadOnlyWrap(), + securityHandlerUnpriviledged, + new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES, + response1.getStatus()); + + // Attempt by a priviledged user should pass. + READ3Response response2 = nfsd.read(xdr_req.asReadOnlyWrap(), + securityHandler, new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK, + response2.getStatus()); + } + + @Test(timeout = 60000) + public void testWrite() throws Exception { + HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar"); + long dirId = status.getFileId(); + FileHandle handle = new FileHandle(dirId); + + byte[] buffer = new byte[10]; + for (int i = 0; i < 10; i++) { + buffer[i] = (byte) i; + } + + WRITE3Request writeReq = new WRITE3Request(handle, 0, 10, + WriteStableHow.DATA_SYNC, ByteBuffer.wrap(buffer)); + XDR xdr_req = new XDR(); + writeReq.serialize(xdr_req); + + // Attempt by an unpriviledged user should fail. + WRITE3Response response1 = nfsd.write(xdr_req.asReadOnlyWrap(), + null, 1, securityHandlerUnpriviledged, + new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES, + response1.getStatus()); + + // Attempt by a priviledged user should pass. + WRITE3Response response2 = nfsd.write(xdr_req.asReadOnlyWrap(), + null, 1, securityHandler, + new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect response:", null, response2); + } + + @Test(timeout = 60000) + public void testCreate() throws Exception { + HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir); + long dirId = status.getFileId(); + XDR xdr_req = new XDR(); + FileHandle handle = new FileHandle(dirId); + handle.serialize(xdr_req); + xdr_req.writeString("fubar"); + xdr_req.writeInt(Nfs3Constant.CREATE_UNCHECKED); + SetAttr3 symAttr = new SetAttr3(); + symAttr.serialize(xdr_req); + + // Attempt by an unpriviledged user should fail. + CREATE3Response response1 = nfsd.create(xdr_req.asReadOnlyWrap(), + securityHandlerUnpriviledged, + new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES, + response1.getStatus()); + + // Attempt by a priviledged user should pass. + CREATE3Response response2 = nfsd.create(xdr_req.asReadOnlyWrap(), + securityHandler, new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK, + response2.getStatus()); + } + + @Test(timeout = 60000) + public void testMkdir() throws Exception { + HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir); + long dirId = status.getFileId(); + XDR xdr_req = new XDR(); + FileHandle handle = new FileHandle(dirId); + handle.serialize(xdr_req); + xdr_req.writeString("fubar"); + SetAttr3 symAttr = new SetAttr3(); + symAttr.serialize(xdr_req); + xdr_req.writeString("bar"); + + // Attempt to remove by an unpriviledged user should fail. + SYMLINK3Response response1 = nfsd.symlink(xdr_req.asReadOnlyWrap(), + securityHandlerUnpriviledged, + new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES, + response1.getStatus()); + + // Attempt to remove by a priviledged user should pass. + SYMLINK3Response response2 = nfsd.symlink(xdr_req.asReadOnlyWrap(), + securityHandler, new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK, + response2.getStatus()); + } + + @Test(timeout = 60000) + public void testSymlink() throws Exception { + HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir); + long dirId = status.getFileId(); + XDR xdr_req = new XDR(); + FileHandle handle = new FileHandle(dirId); + handle.serialize(xdr_req); + xdr_req.writeString("fubar"); + SetAttr3 symAttr = new SetAttr3(); + symAttr.serialize(xdr_req); + xdr_req.writeString("bar"); + + // Attempt by an unpriviledged user should fail. + SYMLINK3Response response1 = nfsd.symlink(xdr_req.asReadOnlyWrap(), + securityHandlerUnpriviledged, + new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES, + response1.getStatus()); + + // Attempt by a priviledged user should pass. + SYMLINK3Response response2 = nfsd.symlink(xdr_req.asReadOnlyWrap(), + securityHandler, new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK, + response2.getStatus()); + } + + @Test(timeout = 60000) + public void testRemove() throws Exception { + HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir); + long dirId = status.getFileId(); + XDR xdr_req = new XDR(); + FileHandle handle = new FileHandle(dirId); + handle.serialize(xdr_req); + xdr_req.writeString("bar"); + + // Attempt by an unpriviledged user should fail. + REMOVE3Response response1 = nfsd.remove(xdr_req.asReadOnlyWrap(), + securityHandlerUnpriviledged, + new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES, + response1.getStatus()); + + // Attempt by a priviledged user should pass. + REMOVE3Response response2 = nfsd.remove(xdr_req.asReadOnlyWrap(), + securityHandler, new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK, + response2.getStatus()); + } + + @Test(timeout = 60000) + public void testRmdir() throws Exception { + HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir); + long dirId = status.getFileId(); + XDR xdr_req = new XDR(); + FileHandle handle = new FileHandle(dirId); + handle.serialize(xdr_req); + xdr_req.writeString("foo"); + + // Attempt by an unpriviledged user should fail. + RMDIR3Response response1 = nfsd.rmdir(xdr_req.asReadOnlyWrap(), + securityHandlerUnpriviledged, + new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES, + response1.getStatus()); + + // Attempt by a priviledged user should pass. + RMDIR3Response response2 = nfsd.rmdir(xdr_req.asReadOnlyWrap(), + securityHandler, new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK, + response2.getStatus()); + } + + @Test(timeout = 60000) + public void testRename() throws Exception { + HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir); + long dirId = status.getFileId(); + XDR xdr_req = new XDR(); + FileHandle handle = new FileHandle(dirId); + handle.serialize(xdr_req); + xdr_req.writeString("bar"); + handle.serialize(xdr_req); + xdr_req.writeString("fubar"); + + // Attempt by an unpriviledged user should fail. + RENAME3Response response1 = nfsd.rename(xdr_req.asReadOnlyWrap(), + securityHandlerUnpriviledged, + new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES, + response1.getStatus()); + + // Attempt by a priviledged user should pass. + RENAME3Response response2 = nfsd.rename(xdr_req.asReadOnlyWrap(), + securityHandler, new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK, + response2.getStatus()); + } + + @Test(timeout = 60000) + public void testReaddir() throws Exception { + HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir); + long dirId = status.getFileId(); + FileHandle handle = new FileHandle(dirId); + XDR xdr_req = new XDR(); + handle.serialize(xdr_req); + xdr_req.writeLongAsHyper(0); + xdr_req.writeLongAsHyper(0); + xdr_req.writeInt(100); + + // Attempt by an unpriviledged user should fail. + READDIR3Response response1 = nfsd.readdir(xdr_req.asReadOnlyWrap(), + securityHandlerUnpriviledged, + new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES, + response1.getStatus()); + + // Attempt by a priviledged user should pass. + READDIR3Response response2 = nfsd.readdir(xdr_req.asReadOnlyWrap(), + securityHandler, new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK, + response2.getStatus()); + } + + @Test(timeout = 60000) + public void testReaddirplus() throws Exception { + HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir); + long dirId = status.getFileId(); + FileHandle handle = new FileHandle(dirId); + XDR xdr_req = new XDR(); + handle.serialize(xdr_req); + xdr_req.writeLongAsHyper(0); + xdr_req.writeLongAsHyper(0); + xdr_req.writeInt(3); + xdr_req.writeInt(2); + + // Attempt by an unpriviledged user should fail. + READDIRPLUS3Response response1 = nfsd.readdirplus(xdr_req.asReadOnlyWrap(), + securityHandlerUnpriviledged, + new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES, + response1.getStatus()); + + // Attempt by a priviledged user should pass. + READDIRPLUS3Response response2 = nfsd.readdirplus(xdr_req.asReadOnlyWrap(), + securityHandler, new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK, + response2.getStatus()); + } + + @Test(timeout = 60000) + public void testFsstat() throws Exception { + HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar"); + long dirId = status.getFileId(); + FileHandle handle = new FileHandle(dirId); + XDR xdr_req = new XDR(); + handle.serialize(xdr_req); + + // Attempt by an unpriviledged user should fail. + FSSTAT3Response response1 = nfsd.fsstat(xdr_req.asReadOnlyWrap(), + securityHandlerUnpriviledged, + new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES, + response1.getStatus()); + + // Attempt by a priviledged user should pass. + FSSTAT3Response response2 = nfsd.fsstat(xdr_req.asReadOnlyWrap(), + securityHandler, new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK, + response2.getStatus()); + } + + @Test(timeout = 60000) + public void testFsinfo() throws Exception { + HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar"); + long dirId = status.getFileId(); + FileHandle handle = new FileHandle(dirId); + XDR xdr_req = new XDR(); + handle.serialize(xdr_req); + + // Attempt by an unpriviledged user should fail. + FSINFO3Response response1 = nfsd.fsinfo(xdr_req.asReadOnlyWrap(), + securityHandlerUnpriviledged, + new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES, + response1.getStatus()); + + // Attempt by a priviledged user should pass. + FSINFO3Response response2 = nfsd.fsinfo(xdr_req.asReadOnlyWrap(), + securityHandler, new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK, + response2.getStatus()); + } + + @Test(timeout = 60000) + public void testPathconf() throws Exception { + HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar"); + long dirId = status.getFileId(); + FileHandle handle = new FileHandle(dirId); + XDR xdr_req = new XDR(); + handle.serialize(xdr_req); + + // Attempt by an unpriviledged user should fail. + PATHCONF3Response response1 = nfsd.pathconf(xdr_req.asReadOnlyWrap(), + securityHandlerUnpriviledged, + new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES, + response1.getStatus()); + + // Attempt by a priviledged user should pass. + PATHCONF3Response response2 = nfsd.pathconf(xdr_req.asReadOnlyWrap(), + securityHandler, new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK, + response2.getStatus()); + } + + @Test(timeout = 60000) + public void testCommit() throws Exception { + HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar"); + long dirId = status.getFileId(); + FileHandle handle = new FileHandle(dirId); + XDR xdr_req = new XDR(); + handle.serialize(xdr_req); + xdr_req.writeLongAsHyper(0); + xdr_req.writeInt(5); + + Channel ch = Mockito.mock(Channel.class); + + // Attempt by an unpriviledged user should fail. + COMMIT3Response response1 = nfsd.commit(xdr_req.asReadOnlyWrap(), + ch, 1, securityHandlerUnpriviledged, + new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES, + response1.getStatus()); + + // Attempt by a priviledged user should pass. + COMMIT3Response response2 = nfsd.commit(xdr_req.asReadOnlyWrap(), + ch, 1, securityHandler, + new InetSocketAddress("localhost", 1234)); + assertEquals("Incorrect COMMIT3Response:", null, response2); + } + @Test(timeout=1000) public void testIdempotent() { Object[][] procedures = { @@ -63,4 +650,41 @@ public class TestRpcProgramNfs3 { } } } + + @Test + public void testDeprecatedKeys() { + NfsConfiguration conf = new NfsConfiguration(); + conf.setInt("nfs3.server.port", 998); + assertTrue(conf.getInt(NfsConfigKeys.DFS_NFS_SERVER_PORT_KEY, 0) == 998); + + conf.setInt("nfs3.mountd.port", 999); + assertTrue(conf.getInt(NfsConfigKeys.DFS_NFS_MOUNTD_PORT_KEY, 0) == 999); + + conf.set("dfs.nfs.exports.allowed.hosts", "host1"); + assertTrue(conf.get(CommonConfigurationKeys.NFS_EXPORTS_ALLOWED_HOSTS_KEY) + .equals("host1")); + + conf.setInt("dfs.nfs.exports.cache.expirytime.millis", 1000); + assertTrue(conf.getInt( + Nfs3Constant.NFS_EXPORTS_CACHE_EXPIRYTIME_MILLIS_KEY, 0) == 1000); + + conf.setInt("hadoop.nfs.userupdate.milly", 10); + assertTrue(conf.getInt(Nfs3Constant.NFS_USERGROUP_UPDATE_MILLIS_KEY, 0) == 10); + + conf.set("dfs.nfs3.dump.dir", "/nfs/tmp"); + assertTrue(conf.get(NfsConfigKeys.DFS_NFS_FILE_DUMP_DIR_KEY).equals( + "/nfs/tmp")); + + conf.setBoolean("dfs.nfs3.enableDump", false); + assertTrue(conf.getBoolean(NfsConfigKeys.DFS_NFS_FILE_DUMP_KEY, true) == false); + + conf.setInt("dfs.nfs3.max.open.files", 500); + assertTrue(conf.getInt(NfsConfigKeys.DFS_NFS_MAX_OPEN_FILES_KEY, 0) == 500); + + conf.setInt("dfs.nfs3.stream.timeout", 6000); + assertTrue(conf.getInt(NfsConfigKeys.DFS_NFS_STREAM_TIMEOUT_KEY, 0) == 6000); + + conf.set("dfs.nfs3.export.point", "/dir1"); + assertTrue(conf.get(NfsConfigKeys.DFS_NFS_EXPORT_POINT_KEY).equals("/dir1")); + } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java Tue Aug 19 23:49:39 2014 @@ -22,16 +22,16 @@ import static org.junit.Assert.assertTru import static org.junit.Assert.fail; import java.io.IOException; -import java.net.InetAddress; +import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.concurrent.ConcurrentNavigableMap; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSClient; -import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; +import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys; +import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration; import org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx.COMMIT_STATUS; import org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx.CommitCtx; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; @@ -50,6 +50,7 @@ import org.apache.hadoop.nfs.nfs3.respon import org.apache.hadoop.nfs.nfs3.response.READ3Response; import org.apache.hadoop.oncrpc.XDR; import org.apache.hadoop.oncrpc.security.SecurityHandler; +import org.apache.hadoop.security.authorize.DefaultImpersonationProvider; import org.apache.hadoop.security.authorize.ProxyUsers; import org.jboss.netty.channel.Channel; import org.junit.Assert; @@ -138,7 +139,7 @@ public class TestWrites { Mockito.when(fos.getPos()).thenReturn((long) 0); OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, - new IdUserGroup()); + new IdUserGroup(new NfsConfiguration())); COMMIT_STATUS ret; @@ -190,6 +191,29 @@ public class TestWrites { ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, false); Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_FINISHED); } + + @Test + public void testCheckCommitAixCompatMode() throws IOException { + DFSClient dfsClient = Mockito.mock(DFSClient.class); + Nfs3FileAttributes attr = new Nfs3FileAttributes(); + HdfsDataOutputStream fos = Mockito.mock(HdfsDataOutputStream.class); + + // Last argument "true" here to enable AIX compatibility mode. + OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, + new IdUserGroup(new NfsConfiguration()), true); + + // Test fall-through to pendingWrites check in the event that commitOffset + // is greater than the number of bytes we've so far flushed. + Mockito.when(fos.getPos()).thenReturn((long) 2); + COMMIT_STATUS status = ctx.checkCommitInternal(5, null, 1, attr, false); + Assert.assertTrue(status == COMMIT_STATUS.COMMIT_FINISHED); + + // Test the case when we actually have received more bytes than we're trying + // to commit. + Mockito.when(fos.getPos()).thenReturn((long) 10); + status = ctx.checkCommitInternal(5, null, 1, attr, false); + Assert.assertTrue(status == COMMIT_STATUS.COMMIT_DO_SYNC); + } @Test // Validate all the commit check return codes OpenFileCtx.COMMIT_STATUS, which @@ -200,13 +224,14 @@ public class TestWrites { Nfs3FileAttributes attr = new Nfs3FileAttributes(); HdfsDataOutputStream fos = Mockito.mock(HdfsDataOutputStream.class); Mockito.when(fos.getPos()).thenReturn((long) 0); + NfsConfiguration config = new NfsConfiguration(); OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, - new IdUserGroup()); + new IdUserGroup(config)); FileHandle h = new FileHandle(1); // fake handle for "/dumpFilePath" COMMIT_STATUS ret; - WriteManager wm = new WriteManager(new IdUserGroup(), new Configuration()); + WriteManager wm = new WriteManager(new IdUserGroup(config), config, false); assertTrue(wm.addOpenFileStream(h, ctx)); // Test inactive open file context @@ -279,7 +304,7 @@ public class TestWrites { @Test public void testWriteStableHow() throws IOException, InterruptedException { - HdfsConfiguration config = new HdfsConfiguration(); + NfsConfiguration config = new NfsConfiguration(); DFSClient client = null; MiniDFSCluster cluster = null; RpcProgramNfs3 nfsd; @@ -288,10 +313,12 @@ public class TestWrites { System.getProperty("user.name")); String currentUser = System.getProperty("user.name"); config.set( - ProxyUsers.getProxySuperuserGroupConfKey(currentUser), + DefaultImpersonationProvider.getTestProvider(). + getProxySuperuserGroupConfKey(currentUser), "*"); config.set( - ProxyUsers.getProxySuperuserIpConfKey(currentUser), + DefaultImpersonationProvider.getTestProvider(). + getProxySuperuserIpConfKey(currentUser), "*"); ProxyUsers.refreshSuperUserGroupsConfiguration(config); @@ -317,7 +344,7 @@ public class TestWrites { XDR createXdr = new XDR(); createReq.serialize(createXdr); CREATE3Response createRsp = nfsd.create(createXdr.asReadOnlyWrap(), - securityHandler, InetAddress.getLocalHost()); + securityHandler, new InetSocketAddress("localhost", 1234)); FileHandle handle = createRsp.getObjHandle(); // Test DATA_SYNC @@ -330,7 +357,7 @@ public class TestWrites { XDR writeXdr = new XDR(); writeReq.serialize(writeXdr); nfsd.write(writeXdr.asReadOnlyWrap(), null, 1, securityHandler, - InetAddress.getLocalHost()); + new InetSocketAddress("localhost", 1234)); waitWrite(nfsd, handle, 60000); @@ -339,7 +366,7 @@ public class TestWrites { XDR readXdr = new XDR(); readReq.serialize(readXdr); READ3Response readRsp = nfsd.read(readXdr.asReadOnlyWrap(), - securityHandler, InetAddress.getLocalHost()); + securityHandler, new InetSocketAddress("localhost", 1234)); assertTrue(Arrays.equals(buffer, readRsp.getData().array())); @@ -351,7 +378,7 @@ public class TestWrites { XDR createXdr2 = new XDR(); createReq2.serialize(createXdr2); CREATE3Response createRsp2 = nfsd.create(createXdr2.asReadOnlyWrap(), - securityHandler, InetAddress.getLocalHost()); + securityHandler, new InetSocketAddress("localhost", 1234)); FileHandle handle2 = createRsp2.getObjHandle(); WRITE3Request writeReq2 = new WRITE3Request(handle2, 0, 10, @@ -359,7 +386,7 @@ public class TestWrites { XDR writeXdr2 = new XDR(); writeReq2.serialize(writeXdr2); nfsd.write(writeXdr2.asReadOnlyWrap(), null, 1, securityHandler, - InetAddress.getLocalHost()); + new InetSocketAddress("localhost", 1234)); waitWrite(nfsd, handle2, 60000); @@ -368,7 +395,7 @@ public class TestWrites { XDR readXdr2 = new XDR(); readReq2.serialize(readXdr2); READ3Response readRsp2 = nfsd.read(readXdr2.asReadOnlyWrap(), - securityHandler, InetAddress.getLocalHost()); + securityHandler, new InetSocketAddress("localhost", 1234)); assertTrue(Arrays.equals(buffer, readRsp2.getData().array())); // FILE_SYNC should sync the file size @@ -381,4 +408,80 @@ public class TestWrites { } } } + + @Test + public void testOOOWrites() throws IOException, InterruptedException { + NfsConfiguration config = new NfsConfiguration(); + MiniDFSCluster cluster = null; + RpcProgramNfs3 nfsd; + final int bufSize = 32; + final int numOOO = 3; + SecurityHandler securityHandler = Mockito.mock(SecurityHandler.class); + Mockito.when(securityHandler.getUser()).thenReturn( + System.getProperty("user.name")); + String currentUser = System.getProperty("user.name"); + config.set( + DefaultImpersonationProvider.getTestProvider(). + getProxySuperuserGroupConfKey(currentUser), + "*"); + config.set( + DefaultImpersonationProvider.getTestProvider(). + getProxySuperuserIpConfKey(currentUser), + "*"); + ProxyUsers.refreshSuperUserGroupsConfiguration(config); + // Use emphral port in case tests are running in parallel + config.setInt("nfs3.mountd.port", 0); + config.setInt("nfs3.server.port", 0); + + try { + cluster = new MiniDFSCluster.Builder(config).numDataNodes(1).build(); + cluster.waitActive(); + + Nfs3 nfs3 = new Nfs3(config); + nfs3.startServiceInternal(false); + nfsd = (RpcProgramNfs3) nfs3.getRpcProgram(); + + DFSClient dfsClient = new DFSClient(NameNode.getAddress(config), config); + HdfsFileStatus status = dfsClient.getFileInfo("/"); + FileHandle rootHandle = new FileHandle(status.getFileId()); + + CREATE3Request createReq = new CREATE3Request(rootHandle, + "out-of-order-write" + System.currentTimeMillis(), + Nfs3Constant.CREATE_UNCHECKED, new SetAttr3(), 0); + XDR createXdr = new XDR(); + createReq.serialize(createXdr); + CREATE3Response createRsp = nfsd.create(createXdr.asReadOnlyWrap(), + securityHandler, new InetSocketAddress("localhost", 1234)); + FileHandle handle = createRsp.getObjHandle(); + + byte[][] oooBuf = new byte[numOOO][bufSize]; + for (int i = 0; i < numOOO; i++) { + Arrays.fill(oooBuf[i], (byte) i); + } + + for (int i = 0; i < numOOO; i++) { + final long offset = (numOOO - 1 - i) * bufSize; + WRITE3Request writeReq = new WRITE3Request(handle, offset, bufSize, + WriteStableHow.UNSTABLE, ByteBuffer.wrap(oooBuf[i])); + XDR writeXdr = new XDR(); + writeReq.serialize(writeXdr); + nfsd.write(writeXdr.asReadOnlyWrap(), null, 1, securityHandler, + new InetSocketAddress("localhost", 1234)); + } + + waitWrite(nfsd, handle, 60000); + READ3Request readReq = new READ3Request(handle, bufSize, bufSize); + XDR readXdr = new XDR(); + readReq.serialize(readXdr); + READ3Response readRsp = nfsd.read(readXdr.asReadOnlyWrap(), + securityHandler, new InetSocketAddress("localhost", config.getInt( + NfsConfigKeys.DFS_NFS_SERVER_PORT_KEY, + NfsConfigKeys.DFS_NFS_SERVER_PORT_DEFAULT))); + assertTrue(Arrays.equals(oooBuf[1], readRsp.getData().array())); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/resources/core-site.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/resources/core-site.xml?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/resources/core-site.xml (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/resources/core-site.xml Tue Aug 19 23:49:39 2014 @@ -18,12 +18,12 @@ - nfs3.server.port + nfs.server.port 2079 - nfs3.mountd.port + nfs.mountd.port 4272 Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Aug 19 23:49:39 2014 @@ -12,6 +12,8 @@ Trunk (Unreleased) HDFS-5570. Deprecate hftp / hsftp and replace them with webhdfs / swebhdfs. (wheat9) + HDFS-2538. option to disable fsck dots (Mohammad Kamrul Islam via aw) + NEW FEATURES HDFS-3125. Add JournalService to enable Journal Daemon. (suresh) @@ -53,8 +55,6 @@ Trunk (Unreleased) HDFS-3030. Remove getProtocolVersion and getProtocolSignature from translators. (jitendra) - HDFS-2976. Remove unnecessary method (tokenRefetchNeeded) in DFSClient. - HDFS-3111. Missing license headers in trunk. (umamahesh) HDFS-3091. Update the usage limitations of ReplaceDatanodeOnFailure policy in @@ -95,8 +95,6 @@ Trunk (Unreleased) HDFS-3768. Exception in TestJettyHelper is incorrect. (Eli Reisman via jghoman) - HDFS-3851. DFSOutputStream class code cleanup. (Jing Zhao via suresh) - HDFS-2580. NameNode#main(...) can make use of GenericOptionsParser. (harsh) HDFS-2127. Add a test that ensure AccessControlExceptions contain @@ -125,10 +123,20 @@ Trunk (Unreleased) HDFS-6228. comments typo fix for FsDatasetImpl.java (zhaoyunjiong via umamahesh) + HDFS-6246. Remove 'dfs.support.append' flag from trunk code. (umamahesh) + + HDFS-6252. Phase out the old web UI in HDFS. (wheat9) + + HDFS-6609. Use DirectorySnapshottableFeature to represent a snapshottable + directory. (Jing Zhao via wheat9) + + HDFS-6482. Use block ID-based block layout on datanodes (James Thomas via + Colin Patrick McCabe) + OPTIMIZATIONS BUG FIXES - + HADOOP-9635 Fix potential Stack Overflow in DomainSocket.c (V. Karthik Kumar via cmccabe) @@ -179,9 +187,6 @@ Trunk (Unreleased) HDFS-3549. Fix dist tar build fails in hadoop-hdfs-raid project. (Jason Lowe via daryn) - HDFS-3482. hdfs balancer throws ArrayIndexOutOfBoundsException - if option is specified without values. ( Madhukara Phatak via umamahesh) - HDFS-3614. Revert unused MiniDFSCluster constructor from HDFS-3049. (acmurthy via eli) @@ -193,9 +198,6 @@ Trunk (Unreleased) HDFS-3834. Remove unused static fields NAME, DESCRIPTION and Usage from Command. (Jing Zhao via suresh) - HADOOP-8158. Interrupting hadoop fs -put from the command line - causes a LeaseExpiredException. (daryn via harsh) - HDFS-2434. TestNameNodeMetrics.testCorruptBlock fails intermittently. (Jing Zhao via suresh) @@ -250,6 +252,274 @@ Trunk (Unreleased) HDFS-5794. Fix the inconsistency of layout version number of ADD_DATANODE_AND_STORAGE_UUIDS between trunk and branch-2. (jing9) + HDFS-6657. Remove link to 'Legacy UI' in trunk's Namenode UI. + (Vinayakumar B via wheat 9) + +Release 2.6.0 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + HDFS-6613. Improve logging in caching classes. (wang) + + HDFS-6511. BlockManager#computeInvalidateWork() could do nothing. (Juan Yu via wang) + + HDFS-6638. Shorten test run time with a smaller retry timeout setting. + (Liang Xie via cnauroth) + + HDFS-6627. Rename DataNode#checkWriteAccess to checkReadAccess. + (Liang Xie via cnauroth) + + HDFS-6645. Add test for successive Snapshots between XAttr modifications. + (Stephen Chu via jing9) + + HDFS-6643. Refactor INodeWithAdditionalFields.PermissionStatusFormat and + INodeFile.HeaderFormat. (szetszwo) + + HDFS-6640. Syntax for MKDIRS, CREATESYMLINK, and SETXATTR are given wrongly + in WebHdfs document (missed webhdfs/v1). (Stephen Chu via jing9) + + HDFS-5202. Support Centralized Cache Management on Windows. (cnauroth) + + HDFS-2976. Remove unnecessary method (tokenRefetchNeeded) in DFSClient. + (Uma Maheswara Rao G) + + HDFS-3851. DFSOutputStream class code cleanup. (Jing Zhao via suresh) + + HDFS-2856. Fix block protocol so that Datanodes don't require root or jsvc. + (cnauroth) + + HDFS-5624. Add HDFS tests for ACLs in combination with viewfs. + (Stephen Chu via cnauroth) + + HDFS-6655. Add 'header banner' to 'explorer.html' also in Namenode UI + (vinayakumarb) + + HDFS-4120. Add a new "-skipSharedEditsCheck" option for BootstrapStandby + (Liang Xie and Rakesh R via vinayakumarb) + + HDFS-6597. Add a new option to NN upgrade to terminate the process after + upgrade on NN is completed. (Danilo Vunjak via cnauroth) + + HDFS-6700. BlockPlacementPolicy shoud choose storage but not datanode for + deletion. (szetszwo) + + HDFS-6616. Add exclude-datanodes feature to WebHDFS redirection so that it + will not redirect retries to the same datanode. (zhaoyunjiong via szetszwo) + + HDFS-6702. Change DFSClient to pass the StorageType from the namenode to + datanodes and change datanode to write block replicas using the specified + storage type. (szetszwo) + + HDFS-6701. Make seed optional in NetworkTopology#sortByDistance. + (Ashwin Shankar via wang) + + HDFS-6755. There is an unnecessary sleep in the code path where + DFSOutputStream#close gives up its attempt to contact the namenode + (mitdesai21 via cmccabe) + + HDFS-6750. The DataNode should use its shared memory segment to mark + short-circuit replicas that have been unlinked as stale (cmccabe) + + HDFS-6739. Add getDatanodeStorageReport to ClientProtocol. (szetszwo) + + HDFS-6665. Add tests for XAttrs in combination with viewfs. + (Stephen Chu via wang) + + HDFS-6778. The extended attributes javadoc should simply refer to the + user docs. (clamb via wang) + + HDFS-6570. add api that enables checking if a user has certain permissions on + a file. (Jitendra Pandey via cnauroth) + + HDFS-6441. Add ability to exclude/include specific datanodes while + balancing. (Benoy Antony and Yu Li via Arpit Agarwal) + + HDFS-6685. Balancer should preserve storage type of replicas. (szetszwo) + + HDFS-6798. Add test case for incorrect data node condition during + balancing. (Benoy Antony via Arpit Agarwal) + + HDFS-6796. Improve the argument check during balancer command line parsing. + (Benoy Antony via szetszwo) + + HDFS-6794. Update BlockManager methods to use DatanodeStorageInfo + where possible (Arpit Agarwal) + + HDFS-6802. Some tests in TestDFSClientFailover are missing @Test + annotation. (Akira Ajisaka via wang) + + HDFS-6788. Improve synchronization in BPOfferService with read write lock. + (Yongjun Zhang via wang) + + HDFS-6787. Remove duplicate code in FSDirectory#unprotectedConcat. (Yi Liu via umamahesh) + + HDFS-6809. Move Balancer's inner classes MovedBlocks and Matcher as to + standalone classes and separates KeyManager from NameNodeConnector. + (szetszwo) + + HDFS-6812. Remove addBlock and replaceBlock from DatanodeDescriptor. + (szetszwo) + + HDFS-6781. Separate HDFS commands from CommandsManual.apt.vm. (Akira + Ajisaka via Arpit Agarwal) + + HDFS-6728. Dynamically add new volumes to DataStorage, formatted if + necessary. (Lei Xu via atm) + + HDFS-6740. Make FSDataset support adding data volumes dynamically. (Lei + Xu via atm) + + HDFS-6722. Display readable last contact time for dead nodes on NN webUI. + (Ming Ma via wheat9) + + HDFS-6772. Get DN storages out of blockContentsStale state faster after + NN restarts. (Ming Ma via Arpit Agarwal) + + HDFS-573. Porting libhdfs to Windows. (cnauroth) + + HDFS-6828. Separate block replica dispatching from Balancer. (szetszwo via + jing9) + + HDFS-6837. Code cleanup for Balancer and Dispatcher. (szetszwo via + jing9) + + HDFS-6838. Code cleanup for unnecessary INode replacement. + (Jing Zhao via wheat9) + + HDFS-6836. HDFS INFO logging is verbose & uses file appenders. (Xiaoyu + Yao via Arpit Agarwal) + + HDFS-6567. Normalize the order of public final in HdfsFileStatus. + (Tassapol Athiapinya via wheat9) + + HDFS-6849. Replace HttpFS custom proxyuser handling with common + implementation. (tucu) + + HDFS-6850. Move NFS out of order write unit tests into TestWrites class. + (Zhe Zhang via atm) + + HDFS-6188. An ip whitelist based implementation of TrustedChannelResolver. + (Benoy Antony via Arpit Agarwal) + + OPTIMIZATIONS + + HDFS-6690. Deduplicate xattr names in memory. (wang) + + BUG FIXES + + HDFS-6823. dfs.web.authentication.kerberos.principal shows up in logs for + insecure HDFS (Allen Wittenauer via raviprak) + + HDFS-6517. Remove hadoop-metrics2.properties from hdfs project (Akira + AJISAKA via aw) + + HDFS-6617. Flake TestDFSZKFailoverController.testManualFailoverWithDFSHAAdmin + due to a long edit log sync op. (Liang Xie via cnauroth) + + HDFS-6646. [ HDFS Rolling Upgrade - Shell ] shutdownDatanode and getDatanodeInfo + usage is missed ( Brahma Reddy Battula via vinayakumarb) + + HDFS-6630. Unable to fetch the block information by Browsing the file system on + Namenode UI through IE9 ( Haohui Mai via vinayakumarb) + + HADOOP-8158. Interrupting hadoop fs -put from the command line + causes a LeaseExpiredException. (daryn via harsh) + + HDFS-6678. MiniDFSCluster may still be partially running after initialization + fails. (cnauroth) + + HDFS-5809. BlockPoolSliceScanner and high speed hdfs appending make + datanode to drop into infinite loop (cmccabe) + + HDFS-6456. NFS should throw error for invalid entry in + dfs.nfs.exports.allowed.hosts (Abhiraj Butala via brandonli) + + HDFS-6689. NFS doesn't return correct lookup access for direcories (brandonli) + + HDFS-6478. RemoteException can't be retried properly for non-HA scenario. + (Ming Ma via jing9) + + HDFS-6693. TestDFSAdminWithHA fails on windows ( vinayakumarb ) + + HDFS-6667. In HDFS HA mode, Distcp/SLive with webhdfs on secure cluster fails + with Client cannot authenticate via:[TOKEN, KERBEROS] error. (jing9) + + HDFS-6704. Fix the command to launch JournalNode in HDFS-HA document. + (Akira AJISAKA via jing9) + + HDFS-6731. Run "hdfs zkfc-formatZK" on a server in a non-namenode will cause + a null pointer exception. (Masatake Iwasaki via brandonli) + + HDFS-6114. Block Scan log rolling will never happen if blocks written + continuously leading to huge size of dncp_block_verification.log.curr + (vinayakumarb via cmccabe) + + HDFS-6455. NFS: Exception should be added in NFS log for invalid separator in + nfs.exports.allowed.hosts. (Abhiraj Butala via brandonli) + + HDFS-6715. Webhdfs wont fail over when it gets java.io.IOException: Namenode + is in startup mode. (jing9) + + HDFS-5919. FileJournalManager doesn't purge empty and corrupt inprogress edits + files (vinayakumarb) + + HDFS-6752. Avoid Address bind errors in TestDatanodeConfig#testMemlockLimit + (vinayakumarb) + + HDFS-6749. FSNamesystem methods should call resolvePath. + (Charles Lamb via cnauroth) + + HDFS-4629. Using com.sun.org.apache.xml.internal.serialize.* in + XmlEditsVisitor.java is JVM vendor specific. Breaks IBM JAVA. + (Amir Sanjar via stevel) + + HDFS-3482. hdfs balancer throws ArrayIndexOutOfBoundsException + if option is specified without values. ( Madhukara Phatak via umamahesh) + + HDFS-6797. DataNode logs wrong layoutversion during upgrade. (Benoy Antony + via Arpit Agarwal) + + HDFS-6810. StorageReport array is initialized with wrong size in + DatanodeDescriptor#getStorageReports. (szetszwo via Arpit Agarwal) + + HDFS-5723. Append failed FINALIZED replica should not be accepted as valid + when that block is underconstruction (vinayakumarb) + + HDFS-5185. DN fails to startup if one of the data dir is full. (vinayakumarb) + + HDFS-6451. NFS should not return NFS3ERR_IO for AccessControlException + (Abhiraj Butala via brandonli) + + HDFS-6717. JIRA HDFS-5804 breaks default nfs-gateway behavior for unsecured config + (brandonli) + + HDFS-6790. DFSUtil Should Use configuration.getPassword for SSL passwords + (Larry McCay via brandonli) + + HDFS-6791. A block could remain under replicated if all of its replicas are on + decommissioned nodes. (Ming Ma via jing9) + + HDFS-6582. Missing null check in RpcProgramNfs3#read(XDR, SecurityHandler) + (Abhiraj Butala via brandonli) + + HDFS-6830. BlockInfo.addStorage fails when DN changes the storage for a + block replica (Arpit Agarwal) + + HDFS-6247. Avoid timeouts for replaceBlock() call by sending intermediate + responses to Balancer (vinayakumarb) + + HDFS-6783. Fix HDFS CacheReplicationMonitor rescan logic. (Yi Liu and Colin Patrick McCabe via umamahesh) + + HDFS-6825. Edit log corruption due to delayed block removal. + (Yongjun Zhang via wang) + + HDFS-6569. OOB message can't be sent to the client when DataNode shuts down for upgrade + (brandonli) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES @@ -260,6 +530,27 @@ Release 2.5.0 - UNRELEASED NEW FEATURES + HDFS-6281. Provide option to use the NFS Gateway without having to use the + Hadoop portmapper. (atm) + + HDFS-5168. Add cross node dependency support to BlockPlacementPolicy. + (Nikola Vujic via szetszwo) + + HDFS-6334. Client failover proxy provider for IP failover based NN HA. + (kihwal) + + HDFS-6406. Add capability for NFS gateway to reject connections from + unprivileged ports. (atm) + + HDFS-2006. Ability to support storing extended attributes per file. + + HDFS-5978. Create a tool to take fsimage and expose read-only WebHDFS API. + (Akira Ajisaka via wheat9) + + HDFS-6278. Create HTML5-based UI for SNN. (wheat9) + + HDFS-6279. Create new index page for JN / DN. (wheat9) + IMPROVEMENTS HDFS-6007. Update documentation about short-circuit local reads (iwasakims @@ -277,9 +568,6 @@ Release 2.5.0 - UNRELEASED HDFS-6158. Clean up dead code for OfflineImageViewer. (wheat9) - HDFS-5978. Create a tool to take fsimage and expose read-only WebHDFS API. - (Akira Ajisaka via wheat9) - HDFS-6164. Remove lsr in OfflineImageViewer. (wheat9) HDFS-6167. Relocate the non-public API classes in the hdfs.client package. @@ -302,10 +590,199 @@ Release 2.5.0 - UNRELEASED HDFS-6219. Proxy superuser configuration should use true client IP for address checks. (daryn via kihwal) + HDFS-6256. Clean up ImageVisitor and SpotCheckImageVisitor. + (Akira Ajisaka via wheat9) + + HDFS-6265. Prepare HDFS codebase for JUnit 4.11. (cnauroth) + + HDFS-5693. Few NN metrics data points were collected via JMX when NN + is under heavy load. (Ming Ma via jing9) + + HDFS-6273. Config options to allow wildcard endpoints for namenode HTTP + and HTTPS servers. (Arpit Agarwal) + + HDFS-6282. Re-add testIncludeByRegistrationName. (cmccabe) + + HDFS-6266. Identify full path for a given INode. (jing9) + + HDFS-6210. Support GETACLSTATUS operation in WebImageViewer. + (Akira Ajisaka via wheat9) + + HDFS-6269. NameNode Audit Log should differentiate between webHDFS open and + HDFS open. (Eric Payne via jeagles) + + HDFS-6304. Consolidate the logic of path resolution in FSDirectory. + (wheat9) + + HDFS-6295. Add "decommissioning" state and node state filtering to + dfsadmin. (wang) + + HDFS-6294. Use INode IDs to avoid conflicts when a file open for write is + renamed. (cmccabe) + + HDFS-6328. Clean up dead code in FSDirectory. (wheat9) + + HDFS-6230. Expose upgrade status through NameNode web UI. + (Mit Desai via wheat9) + + HDFS-6186. Pause deletion of blocks when the namenode starts up. (jing9) + + HDFS-6293. Issues with OIV processing PB-based fsimages. (kihwal) + + HDFS-2949. Add check to active state transition to prevent operator-induced + split brain. (Rushabh S Shah via kihwal) + + HDFS-6287. Add vecsum test of libhdfs read access times (cmccabe) + + HDFS-5683. Better audit log messages for caching operations. + (Abhiraj Butala via wang) + + HDFS-6345. DFS.listCacheDirectives() should allow filtering based on + cache directive ID. (wang) + + HDFS-6432. Add snapshot related APIs to webhdfs. (jing9) + + HDFS-6396. Remove support for ACL feature from INodeSymlink. + (Charles Lamb via wang) + + HDFS-6435. Add support for specifying a static uid/gid mapping for the NFS + gateway. (atm via wang) + + HDFS-6416. Use Time#monotonicNow in OpenFileCtx and OpenFileCtxCatch to + avoid system clock bugs (Abhiraj Butala via brandonli) + + HDFS-6356. Fix typo in DatanodeLayoutVersion. (Tulasi G via wang) + + HDFS-6447. balancer should timestamp the completion message. + (Juan Yu via wang) + + HDFS-6463. Clarify behavior of AclStorage#createFsPermissionForExtendedAcl + in comments. (cnauroth) + + HDFS-6472. fix typo in webapps/hdfs/explorer.js. (Juan Yu via wang) + + HDFS-6056. Clean up NFS config settings (brandonli) + + HDFS-6110 adding more slow action log in critical write path + (Liang Xie via stack) + + HDFS-6109 let sync_file_range() system call run in background + (Liang Xie via stack) + + HDFS-6268. Better sorting in NetworkTopology#pseudoSortByDistance when + no local node is found. (wang) + + HDFS-6369. Document that BlockReader#available() can return more bytes than + are remaining in the block (Ted Yu via Colin Patrick McCabe) + + HDFS-6487. TestStandbyCheckpoint#testSBNCheckpoints is racy. + (Mit Desai via wang) + + HDFS-6297. Add CLI testcases to reflect new features of dfs and dfsadmin + (Dasha Boudnik via cos) + + HDFS-6399. Add note about setfacl in HDFS permissions guide. + (cnauroth via wang) + + HDFS-6315. Decouple recording edit logs from FSDirectory. (wheat9) + + HDFS-6379. HTTPFS - Implement ACLs support. (yoderme via tucu) + + HDFS-6471. Make moveFromLocal CLI testcases to be non-disruptive + (Dasha Boudnik via cos) + + HDFS-6395. Skip checking xattr limits for non-user-visible namespaces. + (Yi Liu via wang). + + HDFS-3493. Invalidate excess corrupted blocks as long as minimum + replication is satisfied. (Juan Yu and Vinayakumar B via wang) + + HDFS-6330. Move mkdirs() to FSNamesystem. (wheat9) + + HDFS-6470. TestBPOfferService.testBPInitErrorHandling is flaky. + (Ming Ma via wang) + + HDFS-6529. Trace logging for RemoteBlockReader2 to identify remote datanode + and file being read. (Anubhav Dhoot via atm) + + HDFS-6499. Use NativeIO#renameTo instead of File#renameTo in + FileJournalManager. (Yongjun Zhang via atm) + + HDFS-6518. TestCacheDirectives#testExceedsCapacity should + take FSN read lock when accessing pendingCached list. + (wang) + + HDFS-6528. Add XAttrs to TestOfflineImageViewer. (Stephen Chu via wang) + + HDFS-6545. Finalizing rolling upgrade can make NN unavailable for a long + duration. (kihwal) + + HDFS-6530. Fix Balancer documentation. (szetszwo) + + HDFS-6480. Move waitForReady() from FSDirectory to FSNamesystem. (wheat9) + + HDFS-6403. Add metrics for log warnings reported by JVM pauses. (Yongjun + Zhang via atm) + + HDFS-6557. Move the reference of fsimage to FSNamesystem. (wheat9) + + HDFS-4667. Capture renamed files/directories in snapshot diff report. (jing9 + and Binglin Chang via jing9) + + HDFS-6507. Improve DFSAdmin to support HA cluster better. + (Zesheng Wu via vinayakumarb) + + HDFS-6578. add toString method to DatanodeStorage for easier debugging. + (Yongjun Zhang via Arpit Agarwal) + + HDFS-6562. Refactor rename() in FSDirectory. (wheat9) + + HDFS-6486. Add user doc for XAttrs via WebHDFS. (Yi Liu via umamahesh) + + HDFS-6430. HTTPFS - Implement XAttr support. (Yi Liu via tucu) + + HDFS-6593. Move SnapshotDiffInfo out of INodeDirectorySnapshottable. + (Jing Zhao via wheat9) + + HDFS-6595. Allow the maximum threads for balancing on datanodes to be + configurable. (Benoy Antony via szetszwo) + + HDFS-6572. Add an option to the NameNode that prints the software and + on-disk image versions. (Charles Lamb via cnauroth) + + HDFS-6603. Add XAttr with ACL test. (Stephen Chu via cnauroth) + + HDFS-6612. MiniDFSNNTopology#simpleFederatedTopology(int) + always hardcode nameservice ID. (Juan Yu via wang) + + HDFS-6614. shorten TestPread run time with a smaller retry timeout setting. + (Liang Xie via cnauroth) + + HDFS-6610. TestShortCircuitLocalRead tests sometimes timeout on slow + machines. (Charles Lamb via wang) + + HDFS-6620. Snapshot docs should specify about preserve options with cp command + (Stephen Chu via umamahesh) + + HDFS-6493. Change dfs.namenode.startup.delay.block.deletion to second + instead of millisecond. (Juan Yu via wang) + + HDFS-6680. BlockPlacementPolicyDefault does not choose favored nodes + correctly. (szetszwo) + + HDFS-6712. Document HDFS Multihoming Settings. (Arpit Agarwal) + OPTIMIZATIONS HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) + HDFS-6460. Ignore stale and decommissioned nodes in + NetworkTopology#sortByDistance. (Yongjun Zhang via wang) + + HDFS-6583. Remove clientNode in FileUnderConstructionFeature. (wheat9) + + HDFS-6599. 2.4 addBlock is 10 to 20 times slower compared to 0.23 (daryn) + BUG FIXES HDFS-6112. NFS Gateway docs are incorrect for allowed hosts configuration. @@ -362,7 +839,312 @@ Release 2.5.0 - UNRELEASED HDFS-6153. Document "fileId" and "childrenNum" fields in the FileStatus Json schema (Akira Ajisaka via vinayakumarb) -Release 2.4.1 - UNRELEASED + HDFS-6178. Decommission on standby NN couldn't finish. (Ming Ma via jing9) + + HDFS-6213. TestDataNodeConfig failing on Jenkins runs due to DN web port + in use. (wang) + + HDFS-6274. Cleanup javadoc warnings in HDFS code. (suresh) + + HDFS-6275. Fix warnings - type arguments can be inferred and redudant + local variable. (suresh) + + HDFS-6217. Webhdfs PUT operations may not work via a http proxy. + (Daryn Sharp via kihwal) + + HDFS-6276. Remove unnecessary conditions and null check. (suresh) + + HDFS-5865. Update OfflineImageViewer document. (Akira Ajisaka via wheat9) + + HDFS-6270. Secondary namenode status page shows transaction count in bytes. + (Benoy Antony via wheat9) + + HDFS-6218. Audit log should use true client IP for proxied webhdfs + operations. (daryn via kihwal) + + HDFS-6288. DFSInputStream Pread doesn't update ReadStatistics. + (Juan Yu via wang) + + HDFS-6289. HA failover can fail if there are pending DN messages for DNs + which no longer exist. (atm) + + HDFS-6337. Setfacl testcase is failing due to dash character in username + in TestAclCLI (umamahesh) + + HDFS-5381. ExtendedBlock#hashCode should use both blockId and block pool ID + (Benoy Antony via Colin Patrick McCabe) + + HDFS-6240. WebImageViewer returns 404 if LISTSTATUS to an empty directory. + (Akira Ajisaka via wheat9) + + HDFS-6351. Command hdfs dfs -rm -r can't remove empty directory. + (Yongjun Zhang via wang) + + HDFS-5522. Datanode disk error check may be incorrectly skipped. + (Rushabh S Shah via kihwal) + + HDFS-6367. EnumSetParam$Domain#parse fails for parameter containing more than one enum. + (Yi Liu via umamahesh) + + HDFS-6305. WebHdfs response decoding may throw RuntimeExceptions (Daryn + Sharp via jeagles) + + HDFS-6355. Fix divide-by-zero, improper use of wall-clock time in + BlockPoolSliceScanner (cmccabe) + + HDFS-6370. Web UI fails to display in intranet under IE. + (Haohui Mai via cnauroth) + + HDFS-6381. Fix a typo in INodeReference.java. (Binglin Chang via jing9) + + HDFS-6400. Cannot execute hdfs oiv_legacy. (Akira AJISAKA via kihwal) + + HDFS-6250. Fix test failed in TestBalancerWithNodeGroup.testBalancerWithRackLocality + (Binglin Chang and Chen He via junping_du) + + HDFS-4913. Deleting file through fuse-dfs when using trash fails requiring + root permissions (cmccabe) + + HDFS-6421. Fix vecsum.c compile on BSD and some other systems. (Mit Desai + via Colin Patrick McCabe) + + HDFS-6419. TestBookKeeperHACheckpoints#TestSBNCheckpoints fails on trunk. + (Akira AJISAKA via kihwal) + + HDFS-6409. Fix typo in log message about NameNode layout version upgrade. + (Chen He via cnauroth) + + HDFS-6433. Replace BytesMoved class with AtomicLong. + (Benoy Antony via cnauroth) + + HDFS-6438. DeleteSnapshot should be a DELETE request in WebHdfs. (jing9) + + HDFS-6423. Diskspace quota usage should be updated when appending data to + partial block. (jing9) + + HDFS-6443. Fix MiniQJMHACluster related test failures. (Zesheng Wu via + Arpit Agarwal) + + HDFS-6227. ShortCircuitCache#unref should purge ShortCircuitReplicas whose + streams have been closed by java interrupts. (Colin Patrick McCabe via jing9) + + HDFS-6442. Fix TestEditLogAutoroll and TestStandbyCheckpoints failure + caused by port conficts. (Zesheng Wu via Arpit Agarwal) + + HDFS-6448. BlockReaderLocalLegacy should set socket timeout based on + conf.socketTimeout (liangxie via cmccabe) + + HDFS-6453. Use Time#monotonicNow to avoid system clock reset. + (Liang Xie via wang) + + HDFS-6461. Use Time#monotonicNow to compute duration in DataNode#shutDown. + (James Thomas via wang) + + HDFS-6462. NFS: fsstat request fails with the secure hdfs (brandonli) + + HDFS-6404. HttpFS should use a 000 umask for mkdir and create + operations. (yoderme via tucu) + + HDFS-6424. blockReport doesn't need to invalidate blocks on SBN. (Ming Ma + via jing9) + + HDFS-6497. Make TestAvailableSpaceVolumeChoosingPolicy deterministic + (cmccabe) + + HDFS-6500. Snapshot shouldn't be removed silently after renaming to an + existing snapshot. (Nicholas SZE via junping_du) + + HDFS-6257. TestCacheDirectives#testExceedsCapacity fails occasionally + (cmccabe) + + HDFS-6364. Incorrect check for unknown datanode in Balancer. (Benoy + Antony via Arpit Agarwal) + + HDFS-6503. Fix typo of DFSAdmin restoreFailedStorage. + (Zesheng Wu via wheat9) + + HDFS-6464. Support multiple xattr.name parameters for WebHDFS getXAttrs. + (Yi Liu via umamahesh) + + HDFS-6539. test_native_mini_dfs is skipped in hadoop-hdfs/pom.xml + (decstery via cmccabe) + + HDFS-6527. Edit log corruption due to defered INode removal. (kihwal and + jing9 via jing9) + + HDFS-6552. add DN storage to a BlockInfo will not replace the different + storage from same DN. (Amir Langer via Arpit Agarwal) + + HDFS-6551. Rename with OVERWRITE option may throw NPE when the target + file/directory is a reference INode. (jing9) + + HDFS-6439. NFS should not reject NFS requests to the NULL procedure whether + port monitoring is enabled or not. (brandonli) + + HDFS-6559. Fix wrong option "dfsadmin -rollingUpgrade start" in the + document. (Akira Ajisaka via Arpit Agarwal) + + HDFS-6553. Add missing DeprecationDeltas for NFS Kerberos configurations + (Stephen Chu via brandonli) + + HDFS-6563. NameNode cannot save fsimage in certain circumstances when + snapshots are in use. (atm) + + HDFS-3848. A Bug in recoverLeaseInternal method of FSNameSystem class + (Hooman Peiro Sajjad and Chen He via kihwal) + + HDFS-6549. Add support for accessing the NFS gateway from the AIX NFS + client. (atm) + + HDFS-6535. HDFS quota update is wrong when file is appended. (George Wong + via jing9) + + HDFS-6222. Remove background token renewer from webhdfs. + (Rushabh Shah and Daryn Sharp via cnauroth) + + HDFS-6580. FSNamesystem.mkdirsInt should call the getAuditFileInfo() + wrapper. (Zhilei Xu via wheat9) + + HDFS-6587. Bug in TestBPOfferService can cause test failure. (Zhilei Xu + via Arpit Agarwal) + + HDFS-6598. Fix a typo in message issued from explorer.js. (Yongjun Zhang + via wheat9) + + HDFS-6475. WebHdfs clients fail without retry because incorrect handling + of StandbyException. (Yongjun Zhang via atm) + + HADOOP-10701. NFS should not validate the access premission only based on + the user's primary group (Harsh J via atm) + + HDFS-6556. Refine XAttr permissions (umamahesh) + + HDFS-6601. Issues in finalizing rolling upgrade when there is a layout + version change (kihwal) + + HDFS-6418. Regression: DFS_NAMENODE_USER_NAME_KEY missing + (szetszwo via stevel) + + HDFS-6558. Missing newline in the description of dfsadmin -rollingUpgrade. + (Chen He via kihwal) + + HDFS-6591. while loop is executed tens of thousands of times in Hedged Read + (Liang Xie via cnauroth) + + HDFS-6604. The short-circuit cache doesn't correctly time out replicas that + haven't been used in a while (cmccabe) + + HDFS-4286. Changes from BOOKKEEPER-203 broken capability of including + bookkeeper-server jar in hidden package of BKJM (Rakesh R via umamahesh) + + HDFS-4221. Remove the format limitation point from BKJM documentation as HDFS-3810 + closed. (Rakesh R via umamahesh) + + HDFS-5411. Update Bookkeeper dependency to 4.2.3. (Rakesh R via umamahesh) + + HDFS-6631. TestPread#testHedgedReadLoopTooManyTimes fails intermittently. + (Liang Xie via cnauroth) + + HDFS-6647. Edit log corruption when pipeline recovery occurs for deleted + file present in snapshot (kihwal) + + HDFS-6378. NFS registration should timeout instead of hanging when + portmap/rpcbind is not available (Abhiraj Butala via brandonli) + + HDFS-6703. NFS: Files can be deleted from a read-only mount + (Srikanth Upputuri via brandonli) + + HDFS-6422. getfattr in CLI doesn't throw exception or return non-0 return code + when xattr doesn't exist. (Charles Lamb via umamahesh) + + HDFS-6696. Name node cannot start if the path of a file under + construction contains ".snapshot". (wang) + + HDFS-6312. WebHdfs HA failover is broken on secure clusters. + (daryn via tucu) + + HDFS-6618. FSNamesystem#delete drops the FSN lock between removing INodes + from the tree and deleting them from the inode map (kihwal via cmccabe) + + HDFS-6622. Rename and AddBlock may race and produce invalid edits (kihwal + via cmccabe) + + HDFS-6723. New NN webUI no longer displays decommissioned state for dead node. + (Ming Ma via wheat9) + + HDFS-6768. Fix a few unit tests that use hard-coded port numbers. (Arpit + Agarwal) + + BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS + + HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh) + + HDFS-6302. Implement XAttr as a INode feature. (Yi Liu via umamahesh) + + HDFS-6309. Javadocs for Xattrs apis in DFSClient and other minor fixups. (Charles Lamb via umamahesh) + + HDFS-6258. Namenode server-side storage for XAttrs. (Yi Liu via umamahesh) + + HDFS-6303. HDFS implementation of FileContext API for XAttrs. (Yi Liu and Charles Lamb via umamahesh) + + HDFS-6324. Shift XAttr helper code out for reuse. (Yi Liu via umamahesh) + + HDFS-6301. NameNode: persist XAttrs in fsimage and record XAttrs modifications to edit log. + (Yi Liu via umamahesh) + + HDFS-6298. XML based End-to-End test for getfattr and setfattr commands. (Yi Liu via umamahesh) + + HDFS-6314. Test cases for XAttrs. (Yi Liu via umamahesh) + + HDFS-6344. Maximum limit on the size of an xattr. (Yi Liu via umamahesh) + + HDFS-6377. Unify xattr name and value limits into a single limit. (wang) + + HDFS-6373. Remove support for extended attributes on symlinks. (Charles Lamb via wang) + + HDFS-6283. Write end user documentation for xattrs. (wang) + + HDFS-6412. Interface audience and stability annotations missing from + several new classes related to xattrs. (wang) + + HDFS-6259. Support extended attributes via WebHDFS. (yliu) + + HDFS-6346. Optimize OP_SET_XATTRS by persisting single Xattr entry per setXattr/removeXattr api call + (Yi Liu via umamahesh) + + HDFS-6331. ClientProtocol#setXattr should not be annotated idempotent. + (umamahesh via wang) + + HDFS-6335. TestOfflineEditsViewer for XAttr. (Yi Liu via umamahesh) + + HDFS-6343. fix TestNamenodeRetryCache and TestRetryCacheWithHA failures. (umamahesh) + + HDFS-6366. FsImage loading failed with RemoveXattr op (umamahesh) + + HDFS-6357. SetXattr should persist rpcIDs for handling retrycache with Namenode restart and HA + (umamahesh) + + HDFS-6372. Handle setXattr rpcIDs for OfflineEditsViewer. (umamahesh) + + HDFS-6410. DFSClient unwraps AclException in xattr methods, but those + methods cannot throw AclException. (wang) + + HDFS-6413. xattr names erroneously handled as case-insensitive. + (Charles Lamb via cnauroth) + + HDFS-6414. xattr modification operations are based on state of latest + snapshot instead of current version of inode. (Andrew Wang via cnauroth) + + HDFS-6374. setXAttr should require the user to be the owner of the file + or directory (Charles Lamb via wang) + + HDFS-6375. Listing extended attributes with the search permission. + (Charles Lamb via wang) + + HDFS-6492. Support create-time xattrs and atomically setting multiple + xattrs. (wang) + +Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES @@ -414,6 +1196,40 @@ Release 2.4.1 - UNRELEASED HDFS-6236. ImageServlet should use Time#monotonicNow to measure latency. (cnauroth) + HDFS-6245. datanode fails to start with a bad disk even when failed + volumes is set. (Arpit Agarwal) + + HDFS-2882. DN continues to start up, even if block pool fails to initialize + (vinayakumarb) + + HDFS-6340. DN can't finalize upgrade. (Rahul Singhal via Arpit Agarwal) + + HDFS-6329. WebHdfs does not work if HA is enabled on NN but logical URI is + not configured. (kihwal) + + HDFS-6313. WebHdfs may use the wrong NN when configured for multiple HA NNs + (kihwal) + + HDFS-6326. WebHdfs ACL compatibility is broken. (cnauroth) + + HDFS-6361. TestIdUserGroup.testUserUpdateSetting failed due to out of range + nfsnobody Id. (Yongjun Zhang via brandonli) + + HDFS-6362. InvalidateBlocks is inconsistent in usage of DatanodeUuid and + StorageID. (Arpit Agarwal) + + HDFS-6402. Suppress findbugs warning for failure to override equals and + hashCode in FsAclPermission. (cnauroth) + + HDFS-6325. Append should fail if the last block has insufficient number of + replicas (Keith Pak via cos) + + HDFS-6397. NN shows inconsistent value in deadnode count. + (Mohammad Kamrul Islam via kihwal) + + HDFS-6411. nfs-hdfs-gateway mount raises I/O error and hangs when a + unauthorized user attempts to access it (brandonli) + Release 2.4.0 - 2014-04-07 INCOMPATIBLE CHANGES