hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1406915 - in /hadoop/common/branches/branch-1: ./ src/core/org/apache/hadoop/ipc/ src/hdfs/org/apache/hadoop/hdfs/ src/test/org/apache/hadoop/hdfs/
Date Thu, 08 Nov 2012 04:03:32 GMT
Author: suresh
Date: Thu Nov  8 04:03:32 2012
New Revision: 1406915

URL: http://svn.apache.org/viewvc?rev=1406915&view=rev
Log:
Reverting revision r1405173 that accidentally committed extraneous change from my workspace

Removed:
    hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/LeaseRenewer.java
Modified:
    hadoop/common/branches/branch-1/CHANGES.txt
    hadoop/common/branches/branch-1/src/core/org/apache/hadoop/ipc/Client.java
    hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java
    hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/AppendTestUtil.java
    hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
    hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestFileAppend4.java
    hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestLeaseRecovery2.java

Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1406915&r1=1406914&r2=1406915&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Thu Nov  8 04:03:32 2012
@@ -307,10 +307,6 @@ Release 1.2.0 - unreleased
     MAPREDUCE-1806. CombineFileInputFormat does not work with paths not on 
     default FS. (Gera Shegalov via tucu)
 
-    HDFS-3791. HDFS-173 Backport - Namenode will not block until a large 
-    directory deletion completes. It allows other operations when the 
-    deletion is in progress. (umamahesh via suresh)
-
     MAPREDUCE-4765. Restarting the JobTracker programmatically can cause
     DelegationTokenRenewal to throw an exception. (rkanter via tucu)
 
@@ -326,6 +322,10 @@ Release 1.1.1 - Unreleased
 
   BUG FIXES
 
+    HDFS-3791. HDFS-173 Backport - Namenode will not block until a large 
+    directory deletion completes. It allows other operations when the 
+    deletion is in progress. (umamahesh via suresh)
+
 Release 1.1.0 - 2012.09.28
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/branch-1/src/core/org/apache/hadoop/ipc/Client.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/core/org/apache/hadoop/ipc/Client.java?rev=1406915&r1=1406914&r2=1406915&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/core/org/apache/hadoop/ipc/Client.java (original)
+++ hadoop/common/branches/branch-1/src/core/org/apache/hadoop/ipc/Client.java Thu Nov  8
04:03:32 2012
@@ -109,7 +109,7 @@ public class Client {
    * @param conf Configuration
    * @return the ping interval
    */
-  public final static int getPingInterval(Configuration conf) {
+  final static int getPingInterval(Configuration conf) {
     return conf.getInt(PING_INTERVAL_NAME, DEFAULT_PING_INTERVAL);
   }
   

Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java?rev=1406915&r1=1406914&r2=1406915&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java (original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java Thu Nov
 8 04:03:32 2012
@@ -86,7 +86,7 @@ public class DFSClient implements FSCons
   volatile boolean clientRunning = true;
   Random r = new Random();
   final String clientName;
-  final LeaseRenewer leaserenewer;
+  final LeaseChecker leasechecker = new LeaseChecker();
   private Configuration conf;
   private long defaultBlockSize;
   private short defaultReplication;
@@ -250,9 +250,6 @@ public class DFSClient implements FSCons
     // dfs.write.packet.size is an internal config variable
     this.writePacketSize = conf.getInt("dfs.write.packet.size", 64*1024);
     this.maxBlockAcquireFailures = getMaxBlockAcquireFailures(conf);
-    
-    // TODO: review this
-    leaserenewer = new LeaseRenewer(this, Client.getPingInterval(conf));
 
     ugi = UserGroupInformation.getCurrentUser();
 
@@ -320,10 +317,10 @@ public class DFSClient implements FSCons
    */
   public synchronized void close() throws IOException {
     if(clientRunning) {
-      leaserenewer.close();
+      leasechecker.close();
       clientRunning = false;
       try {
-        leaserenewer.interruptAndJoin();
+        leasechecker.interruptAndJoin();
       } catch (InterruptedException ie) {
       }
   
@@ -763,7 +760,7 @@ public class DFSClient implements FSCons
     OutputStream result = new DFSOutputStream(src, masked,
         overwrite, createParent, replication, blockSize, progress, buffersize,
         conf.getInt("io.bytes.per.checksum", 512));
-    leaserenewer.put(src, result);
+    leasechecker.put(src, result);
     return result;
   }
 
@@ -818,7 +815,7 @@ public class DFSClient implements FSCons
     }
     final DFSOutputStream result = new DFSOutputStream(src, buffersize, progress,
         lastBlock, stat, conf.getInt("io.bytes.per.checksum", 512));
-    leaserenewer.put(src, result);
+    leasechecker.put(src, result);
     return result;
   }
 
@@ -1395,8 +1392,115 @@ public class DFSClient implements FSCons
     throw new IOException("No live nodes contain current block");
   }
 
-  boolean isLeaseRenewerStarted() {
-    return leaserenewer.isRunning();
+  boolean isLeaseCheckerStarted() {
+    return leasechecker.daemon != null;
+  }
+
+  /** Lease management*/
+  class LeaseChecker implements Runnable {
+    /** A map from src -> DFSOutputStream of files that are currently being
+     * written by this client.
+     */
+    private final SortedMap<String, OutputStream> pendingCreates
+        = new TreeMap<String, OutputStream>();
+
+    private Daemon daemon = null;
+    
+    synchronized void put(String src, OutputStream out) {
+      if (clientRunning) {
+        if (daemon == null) {
+          daemon = new Daemon(this);
+          daemon.start();
+        }
+        pendingCreates.put(src, out);
+      }
+    }
+    
+    synchronized void remove(String src) {
+      pendingCreates.remove(src);
+    }
+    
+    void interruptAndJoin() throws InterruptedException {
+      Daemon daemonCopy = null;
+      synchronized (this) {
+        if (daemon != null) {
+          daemon.interrupt();
+          daemonCopy = daemon;
+        }
+      }
+     
+      if (daemonCopy != null) {
+        LOG.debug("Wait for lease checker to terminate");
+        daemonCopy.join();
+      }
+    }
+
+    void close() {
+      while (true) {
+        String src;
+        OutputStream out;
+        synchronized (this) {
+          if (pendingCreates.isEmpty()) {
+            return;
+          }
+          src = pendingCreates.firstKey();
+          out = pendingCreates.remove(src);
+        }
+        if (out != null) {
+          try {
+            out.close();
+          } catch (IOException ie) {
+            LOG.error("Exception closing file " + src+ " : " + ie, ie);
+          }
+        }
+      }
+    }
+
+    private void renew() throws IOException {
+      synchronized(this) {
+        if (pendingCreates.isEmpty()) {
+          return;
+        }
+      }
+      namenode.renewLease(clientName);
+    }
+
+    /**
+     * Periodically check in with the namenode and renew all the leases
+     * when the lease period is half over.
+     */
+    public void run() {
+      long lastRenewed = 0;
+      while (clientRunning && !Thread.interrupted()) {
+        if (System.currentTimeMillis() - lastRenewed > (LEASE_SOFTLIMIT_PERIOD / 2)) {
+          try {
+            renew();
+            lastRenewed = System.currentTimeMillis();
+          } catch (IOException ie) {
+            LOG.warn("Problem renewing lease for " + clientName, ie);
+          }
+        }
+
+        try {
+          Thread.sleep(1000);
+        } catch (InterruptedException ie) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug(this + " is interrupted.", ie);
+          }
+          return;
+        }
+      }
+    }
+
+    /** {@inheritDoc} */
+    public String toString() {
+      String s = getClass().getSimpleName();
+      if (LOG.isTraceEnabled()) {
+        return s + "@" + DFSClient.this + ": "
+               + StringUtils.stringifyException(new Throwable("for testing"));
+      }
+      return s;
+    }
   }
 
   /** Utility class to encapsulate data node info and its address. */
@@ -3890,7 +3994,7 @@ public class DFSClient implements FSCons
           throw e;
       }
       closeInternal();
-      leaserenewer.remove(src);
+      leasechecker.remove(src);
       
       if (s != null) {
         s.close();
@@ -3907,7 +4011,7 @@ public class DFSClient implements FSCons
       response.close();
       closed = true;
     }
-    
+ 
     // shutdown datastreamer and responseprocessor threads.
     private void closeThreads() throws IOException {
       try {

Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/AppendTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/AppendTestUtil.java?rev=1406915&r1=1406914&r2=1406915&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/AppendTestUtil.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/AppendTestUtil.java Thu
Nov  8 04:03:32 2012
@@ -165,7 +165,7 @@ public class AppendTestUtil {
     LOG.info("leasechecker.interruptAndJoin()");
     // lose the lease on the client
     DistributedFileSystem dfs = (DistributedFileSystem)whichfs;
-    dfs.dfs.leaserenewer.interruptAndJoin();
+    dfs.dfs.leasechecker.interruptAndJoin();
   }
   
   public static void recoverFile(MiniDFSCluster cluster, FileSystem fs,

Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestDistributedFileSystem.java?rev=1406915&r1=1406914&r2=1406915&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
(original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
Thu Nov  8 04:03:32 2012
@@ -111,31 +111,31 @@ public class TestDistributedFileSystem {
 
       {
         DistributedFileSystem dfs = (DistributedFileSystem)cluster.getFileSystem();
-        assertFalse(dfs.dfs.isLeaseRenewerStarted());
+        assertFalse(dfs.dfs.isLeaseCheckerStarted());
   
         //create a file
         FSDataOutputStream out = dfs.create(filepath);
-        assertTrue(dfs.dfs.isLeaseRenewerStarted());
+        assertTrue(dfs.dfs.isLeaseCheckerStarted());
   
         //write something and close
         out.writeLong(millis);
-        assertTrue(dfs.dfs.isLeaseRenewerStarted());
+        assertTrue(dfs.dfs.isLeaseCheckerStarted());
         out.close();
-        assertTrue(dfs.dfs.isLeaseRenewerStarted());
+        assertTrue(dfs.dfs.isLeaseCheckerStarted());
         dfs.close();
       }
 
       {
         DistributedFileSystem dfs = (DistributedFileSystem)cluster.getFileSystem();
-        assertFalse(dfs.dfs.isLeaseRenewerStarted());
+        assertFalse(dfs.dfs.isLeaseCheckerStarted());
 
         //open and check the file
         FSDataInputStream in = dfs.open(filepath);
-        assertFalse(dfs.dfs.isLeaseRenewerStarted());
+        assertFalse(dfs.dfs.isLeaseCheckerStarted());
         assertEquals(millis, in.readLong());
-        assertFalse(dfs.dfs.isLeaseRenewerStarted());
+        assertFalse(dfs.dfs.isLeaseCheckerStarted());
         in.close();
-        assertFalse(dfs.dfs.isLeaseRenewerStarted());
+        assertFalse(dfs.dfs.isLeaseCheckerStarted());
         dfs.close();
       }
     }

Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestFileAppend4.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestFileAppend4.java?rev=1406915&r1=1406914&r2=1406915&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestFileAppend4.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestFileAppend4.java Thu
Nov  8 04:03:32 2012
@@ -656,7 +656,7 @@ public class TestFileAppend4 extends Tes
       // has not been completed in the NN.
       // Lose the leases
       LOG.info("Killing lease checker");
-      client.leaserenewer.interruptAndJoin();
+      client.leasechecker.interruptAndJoin();
 
       FileSystem fs1 = cluster.getFileSystem();
       FileSystem fs2 = AppendTestUtil.createHdfsWithDifferentUsername(
@@ -726,7 +726,7 @@ public class TestFileAppend4 extends Tes
       // has not been completed in the NN.
       // Lose the leases
       LOG.info("Killing lease checker");
-      client.leaserenewer.interruptAndJoin();
+      client.leasechecker.interruptAndJoin();
 
       FileSystem fs1 = cluster.getFileSystem();
       FileSystem fs2 = AppendTestUtil.createHdfsWithDifferentUsername(

Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestLeaseRecovery2.java?rev=1406915&r1=1406914&r2=1406915&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
(original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
Thu Nov  8 04:03:32 2012
@@ -157,7 +157,7 @@ public class TestLeaseRecovery2 extends 
     stm.sync();
     if (triggerSoftLease) {
       AppendTestUtil.LOG.info("leasechecker.interruptAndJoin()");
-      dfs.dfs.leaserenewer.interruptAndJoin();
+      dfs.dfs.leasechecker.interruptAndJoin();
     }
     return filepath;
   }



Mime
View raw message