Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id CE64C7423 for ; Sat, 3 Sep 2011 00:40:38 +0000 (UTC) Received: (qmail 89048 invoked by uid 500); 3 Sep 2011 00:40:38 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 88991 invoked by uid 500); 3 Sep 2011 00:40:38 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 88984 invoked by uid 99); 3 Sep 2011 00:40:37 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 03 Sep 2011 00:40:37 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 03 Sep 2011 00:40:33 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id D0576238897A for ; Sat, 3 Sep 2011 00:40:11 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1164774 - in /hadoop/common/branches/branch-0.20-security: ./ src/hdfs/org/apache/hadoop/hdfs/ src/hdfs/org/apache/hadoop/hdfs/protocol/ src/hdfs/org/apache/hadoop/hdfs/server/namenode/ src/test/org/apache/hadoop/hdfs/ Date: Sat, 03 Sep 2011 00:40:11 -0000 To: common-commits@hadoop.apache.org From: suresh@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110903004011.D0576238897A@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: suresh Date: Sat Sep 3 00:40:10 2011 New Revision: 1164774 URL: http://svn.apache.org/viewvc?rev=1164774&view=rev Log: Port from 0.20-append - HDFS-1554. New semantics for recoverLease. Contributed by Hairong Kuang. Modified: hadoop/common/branches/branch-0.20-security/CHANGES.txt hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/TestDFSClientRetries.java hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/TestLeaseRecovery2.java Modified: hadoop/common/branches/branch-0.20-security/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/CHANGES.txt?rev=1164774&r1=1164773&r2=1164774&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.20-security/CHANGES.txt Sat Sep 3 00:40:10 2011 @@ -92,6 +92,8 @@ Release 0.20.205.0 - unreleased HDFS-1555. Disallow pipelien recovery if a file is already being lease recovered. (hairong) + HDFS-1554. New semantics for recoverLease. (hairong) + IMPROVEMENTS MAPREDUCE-2187. Reporter sends progress during sort/merge. (Anupam Seth via Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java?rev=1164774&r1=1164773&r2=1164774&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java (original) +++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java Sat Sep 3 00:40:10 2011 @@ -554,13 +554,14 @@ public class DFSClient implements FSCons /** * Recover a file's lease * @param src a file's path + * @return true if the file is already closed * @throws IOException */ - void recoverLease(String src) throws IOException { + boolean recoverLease(String src) throws IOException { checkOpen(); try { - namenode.recoverLease(src, clientName); + return namenode.recoverLease(src, clientName); } catch (RemoteException re) { throw re.unwrapRemoteException(FileNotFoundException.class, AccessControlException.class); Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java?rev=1164774&r1=1164773&r2=1164774&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java (original) +++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java Sat Sep 3 00:40:10 2011 @@ -189,12 +189,14 @@ public class DistributedFileSystem exten } /** - * Trigger the lease reovery of a file + * Start the lease recovery of a file + * * @param f a file + * @return true if the file is already closed * @throws IOException if an error occurs */ - public void recoverLease(Path f) throws IOException { - dfs.recoverLease(getPathName(f)); + public boolean recoverLease(Path f) throws IOException { + return dfs.recoverLease(getPathName(f)); } /** This optional operation is not yet supported. */ Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java?rev=1164774&r1=1164773&r2=1164774&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java (original) +++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java Sat Sep 3 00:40:10 2011 @@ -50,9 +50,9 @@ public interface ClientProtocol extends * Compared to the previous version the following changes have been introduced: * (Only the latest change is reflected. * The log of historical changes can be retrieved from the svn). - * 62: Introduce a lightweight recoverLease RPC + * 63: recoverLease return if the file is closed or not */ - public static final long versionID = 62L; + public static final long versionID = 63L; /////////////////////////////////////// // File contents @@ -134,12 +134,14 @@ public interface ClientProtocol extends public LocatedBlock append(String src, String clientName) throws IOException; /** - * Trigger lease recovery to happen - * @param src path of the file to trigger lease recovery + * Start lease recovery + * + * @param src path of the file to start lease recovery * @param clientName name of the current client + * @return true if the file is already closed * @throws IOException */ - public void recoverLease(String src, String clientName) throws IOException; + public boolean recoverLease(String src, String clientName) throws IOException; /** * Set replication for an existing file. Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1164774&r1=1164773&r2=1164774&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original) +++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Sat Sep 3 00:40:10 2011 @@ -1216,7 +1216,7 @@ public class FSNamesystem implements FSC try { INode myFile = dir.getFileINode(src); - recoverLeaseInternal(myFile, src, holder, clientMachine); + recoverLeaseInternal(myFile, src, holder, clientMachine, false); try { verifyReplication(src, replication, clientMachine); @@ -1291,16 +1291,17 @@ public class FSNamesystem implements FSC } /** - * Trigger to recover lease; - * When the method returns successfully, the lease has been recovered and - * the file is closed. + * Recover lease; + * Immediately revoke the lease of the current lease holder and start lease + * recovery so that the file can be forced to be closed. * - * @param src the path of the file to trigger release + * @param src the path of the file to start lease recovery * @param holder the lease holder's name * @param clientMachine the client machine's name + * @return true if the file is already closed * @throws IOException */ - synchronized void recoverLease(String src, String holder, String clientMachine) + synchronized boolean recoverLease(String src, String holder, String clientMachine) throws IOException { if (isInSafeMode()) { throw new SafeModeException( @@ -1315,15 +1316,19 @@ public class FSNamesystem implements FSC throw new FileNotFoundException("File not found " + src); } + if (!inode.isUnderConstruction()) { + return true; + } if (isPermissionEnabled) { checkPathAccess(src, FsAction.WRITE); } - recoverLeaseInternal(inode, src, holder, clientMachine); + recoverLeaseInternal(inode, src, holder, clientMachine, true); + return false; } private void recoverLeaseInternal(INode fileInode, - String src, String holder, String clientMachine) + String src, String holder, String clientMachine, boolean force) throws IOException { if (fileInode != null && fileInode.isUnderConstruction()) { INodeFileUnderConstruction pendingFile = (INodeFileUnderConstruction) fileInode; @@ -1336,7 +1341,7 @@ public class FSNamesystem implements FSC // We found the lease for this file. And surprisingly the original // holder is trying to recreate this file. This should never occur. // - if (lease != null) { + if (!force && lease != null) { Lease leaseFile = leaseManager.getLeaseByPath(src); if (leaseFile != null && leaseFile.equals(lease)) { throw new AlreadyBeingCreatedException( @@ -1355,21 +1360,29 @@ public class FSNamesystem implements FSC " on client " + clientMachine + " because pendingCreates is non-null but no leases found."); } - // - // If the original holder has not renewed in the last SOFTLIMIT - // period, then start lease recovery. - // - if (lease.expiredSoftLimit()) { - LOG.info("startFile: recover lease " + lease + ", src=" + src + + if (force) { + // close now: no need to wait for soft lease expiration and + // close only the file src + LOG.info("recoverLease: recover lease " + lease + ", src=" + src + " from client " + pendingFile.clientName); - internalReleaseLease(lease, src); + internalReleaseLeaseOne(lease, src); + } else { + // + // If the original holder has not renewed in the last SOFTLIMIT + // period, then start lease recovery. + // + if (lease.expiredSoftLimit()) { + LOG.info("startFile: recover lease " + lease + ", src=" + src + + " from client " + pendingFile.clientName); + internalReleaseLease(lease, src); + } + throw new AlreadyBeingCreatedException( + "failed to create file " + src + " for " + holder + + " on client " + clientMachine + + ", because this file is already being created by " + + pendingFile.getClientName() + + " on " + pendingFile.getClientMachine()); } - throw new AlreadyBeingCreatedException( - "failed to create file " + src + " for " + holder + - " on client " + clientMachine + - ", because this file is already being created by " + - pendingFile.getClientName() + - " on " + pendingFile.getClientMachine()); } } Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1164774&r1=1164773&r2=1164774&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original) +++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java Sat Sep 3 00:40:10 2011 @@ -575,9 +575,9 @@ public class NameNode implements ClientP } /** {@inheritDoc} */ - public void recoverLease(String src, String clientName) throws IOException { + public boolean recoverLease(String src, String clientName) throws IOException { String clientMachine = getClientMachine(); - namesystem.recoverLease(src, clientName, clientMachine); + return namesystem.recoverLease(src, clientName, clientMachine); } /** {@inheritDoc} */ Modified: hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/TestDFSClientRetries.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/TestDFSClientRetries.java?rev=1164774&r1=1164773&r2=1164774&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/TestDFSClientRetries.java (original) +++ hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/TestDFSClientRetries.java Sat Sep 3 00:40:10 2011 @@ -289,7 +289,7 @@ public class TestDFSClientRetries extend public void setTimes(String src, long mtime, long atime) throws IOException {} - public void recoverLease(String src, String clientName) throws IOException {} + public boolean recoverLease(String src, String clientName) throws IOException {return true;} public Token getDelegationToken(Text renewer) throws IOException { Modified: hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/TestLeaseRecovery2.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/TestLeaseRecovery2.java?rev=1164774&r1=1164773&r2=1164774&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/TestLeaseRecovery2.java (original) +++ hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/TestLeaseRecovery2.java Sat Sep 3 00:40:10 2011 @@ -75,7 +75,7 @@ public class TestLeaseRecovery2 extends //create a file DistributedFileSystem dfs = (DistributedFileSystem)cluster.getFileSystem(); int size = AppendTestUtil.nextInt(FILE_SIZE); - Path filepath = createFile(dfs, size); + Path filepath = createFile(dfs, size, true); // set the soft limit to be 1 second so that the // namenode triggers lease recovery on next attempt to write-for-open. @@ -85,60 +85,61 @@ public class TestLeaseRecovery2 extends verifyFile(dfs, filepath, actual, size); //test recoverLease + // set the soft limit to be 1 hour but recoverLease should + // close the file immediately + cluster.setLeasePeriod(hardLease, hardLease); size = AppendTestUtil.nextInt(FILE_SIZE); - filepath = createFile(dfs, size); + filepath = createFile(dfs, size, false); - // set the soft limit to be 1 second so that the - // namenode triggers lease recovery on next attempt to write-for-open. - cluster.setLeasePeriod(softLease, hardLease); + // test recoverLese from a different client + recoverLease(filepath, null); + verifyFile(dfs, filepath, actual, size); - recoverLease(filepath); + // test recoverlease from the same client + size = AppendTestUtil.nextInt(FILE_SIZE); + filepath = createFile(dfs, size, false); + + // create another file using the same client + Path filepath1 = new Path("/foo" + AppendTestUtil.nextInt()); + FSDataOutputStream stm = dfs.create(filepath1, true, + bufferSize, REPLICATION_NUM, BLOCK_SIZE); + + // recover the first file + recoverLease(filepath, dfs); verifyFile(dfs, filepath, actual, size); + // continue to write to the second file + stm.write(buffer, 0, size); + stm.close(); + verifyFile(dfs, filepath1, actual, size); } finally { try { - if (cluster != null) {cluster.shutdown();} + if (cluster != null) {cluster.getFileSystem().close();cluster.shutdown();} } catch (Exception e) { // ignore } } } - private void recoverLease(Path filepath) throws IOException, InterruptedException { - UserGroupInformation ugi = - UserGroupInformation.createUserForTesting(fakeUsername, - new String [] { fakeGroup}); - - DistributedFileSystem dfs2 = (DistributedFileSystem) - DFSTestUtil.getFileSystemAs(ugi, conf); - - boolean done = false; - while (!done) { - try { - dfs2.recoverLease(filepath); - done = true; - } catch (IOException ioe) { - final String message = ioe.getMessage(); - if (message.contains(AlreadyBeingCreatedException.class.getSimpleName())) { - AppendTestUtil.LOG.info("GOOD! got " + message); - } - else { - AppendTestUtil.LOG.warn("UNEXPECTED IOException", ioe); - } - } - - if (!done) { - AppendTestUtil.LOG.info("sleep " + 1000 + "ms"); - try {Thread.sleep(5000);} catch (InterruptedException e) {} - } + private void recoverLease(Path filepath, DistributedFileSystem dfs2) throws Exception { + if (dfs2==null) { + UserGroupInformation ugi = + UserGroupInformation.createUserForTesting(fakeUsername, + new String [] { fakeGroup}); + dfs2 = (DistributedFileSystem) DFSTestUtil.getFileSystemAs(ugi, conf); + } + + while (!dfs2.recoverLease(filepath)) { + AppendTestUtil.LOG.info("sleep " + 5000 + "ms"); + Thread.sleep(5000); } } // try to re-open the file before closing the previous handle. This // should fail but will trigger lease recovery. - private Path createFile(DistributedFileSystem dfs, int size) - throws IOException, InterruptedException { + private Path createFile(DistributedFileSystem dfs, int size, + boolean triggerSoftLease) throws IOException, InterruptedException { // create a random file name String filestr = "/foo" + AppendTestUtil.nextInt(); System.out.println("filestr=" + filestr); @@ -154,8 +155,10 @@ public class TestLeaseRecovery2 extends // sync file AppendTestUtil.LOG.info("sync"); stm.sync(); - AppendTestUtil.LOG.info("leasechecker.interruptAndJoin()"); - dfs.dfs.leasechecker.interruptAndJoin(); + if (triggerSoftLease) { + AppendTestUtil.LOG.info("leasechecker.interruptAndJoin()"); + dfs.dfs.leasechecker.interruptAndJoin(); + } return filepath; }