hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1445963 - in /hadoop/common/branches/branch-1: ./ src/core/org/apache/hadoop/security/token/delegation/ src/hdfs/org/apache/hadoop/hdfs/security/token/delegation/ src/test/org/apache/hadoop/hdfs/security/
Date Wed, 13 Feb 2013 22:41:15 GMT
Author: suresh
Date: Wed Feb 13 22:41:15 2013
New Revision: 1445963

URL: http://svn.apache.org/r1445963
Log:
HDFS-4466. Remove the deadlock from AbstractDelegationTokenSecretManager. Contributed by Brandon
Li.

Modified:
    hadoop/common/branches/branch-1/CHANGES.txt
    hadoop/common/branches/branch-1/src/core/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
    hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
    hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/security/TestDelegationToken.java

Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1445963&r1=1445962&r2=1445963&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Wed Feb 13 22:41:15 2013
@@ -487,6 +487,9 @@ Release 1.2.0 - unreleased
     MAPREDUCE-4643. Make job-history cleanup-period configurable. 
     (sandyr via tucu)
 
+    HDFS-4466. Remove the deadlock from AbstractDelegationTokenSecretManager.
+    (Brandon Li via suresh)
+
 Release 1.1.2 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/branch-1/src/core/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/core/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java?rev=1445963&r1=1445962&r2=1445963&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/core/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
(original)
+++ hadoop/common/branches/branch-1/src/core/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
Wed Feb 13 22:41:15 2013
@@ -85,6 +85,12 @@ extends AbstractDelegationTokenIdentifie
   private Thread tokenRemoverThread;
   protected volatile boolean running;
 
+  /**
+   * If the delegation token update thread holds this lock, it will
+   * not get interrupted.
+   */
+  protected Object noInterruptsLock = new Object();
+
   public AbstractDelegationTokenSecretManager(long delegationKeyUpdateInterval,
       long delegationTokenMaxLifetime, long delegationTokenRenewInterval,
       long delegationTokenRemoverScanInterval) {
@@ -360,12 +366,22 @@ extends AbstractDelegationTokenIdentifie
     }
   }
 
-  public synchronized void stopThreads() {
+
+  public void stopThreads() {
     if (LOG.isDebugEnabled())
       LOG.debug("Stopping expired delegation token remover thread");
     running = false;
+
     if (tokenRemoverThread != null) {
-      tokenRemoverThread.interrupt();
+      synchronized (noInterruptsLock) {
+        tokenRemoverThread.interrupt();
+      }
+
+      try {
+        tokenRemoverThread.join();
+      } catch (InterruptedException e) {
+        throw new RuntimeException("Unable to join on token removal thread", e);
+      }
     }
   }
   
@@ -381,24 +397,20 @@ extends AbstractDelegationTokenIdentifie
         while (running) {
           long now = System.currentTimeMillis();
           if (lastMasterKeyUpdate + keyUpdateInterval < now) {
-            synchronized (AbstractDelegationTokenSecretManager.this) {
-              if (running) {
-                try {
-                  rollMasterKey();
-                  lastMasterKeyUpdate = now;
-                } catch (IOException e) {
-                  LOG.error("Master key updating failed. "
-                            + StringUtils.stringifyException(e));
-                }
+            if (running) {
+              try {
+                rollMasterKey();
+                lastMasterKeyUpdate = now;
+              } catch (IOException e) {
+                LOG.error("Master key updating failed. "
+                    + StringUtils.stringifyException(e));
               }
             }
           }
           if (lastTokenCacheCleanup + tokenRemoverScanInterval < now) {
-            synchronized (AbstractDelegationTokenSecretManager.this) {
-              if (running) {
-                removeExpiredToken();
-                lastTokenCacheCleanup = now;
-              }
+            if (running) {
+              removeExpiredToken();
+              lastTokenCacheCleanup = now;
             }
           }
           try {

Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java?rev=1445963&r1=1445962&r2=1445963&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
(original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
Wed Feb 13 22:41:15 2013
@@ -22,6 +22,7 @@ package org.apache.hadoop.hdfs.security.
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.io.InterruptedIOException;
 import java.util.Iterator;
 
 import org.apache.commons.logging.Log;
@@ -283,7 +284,18 @@ public class DelegationTokenSecretManage
   @Override //AbstractDelegationTokenManager
   protected void logUpdateMasterKey(DelegationKey key)
       throws IOException {
-    namesystem.logUpdateMasterKey(key);
+    synchronized (noInterruptsLock) {
+      // The edit logging code will fail catastrophically if it
+      // is interrupted during a logSync, since the interrupt
+      // closes the edit log files. Doing this inside the
+      // above lock and then checking interruption status
+      // prevents this bug.
+      if (Thread.interrupted()) {
+        throw new InterruptedIOException(
+            "Interrupted before updating master key");
+      }
+      namesystem.logUpdateMasterKey(key);
+    }
   }
 
   /** A utility method for creating credentials. */

Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/security/TestDelegationToken.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/security/TestDelegationToken.java?rev=1445963&r1=1445962&r2=1445963&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/security/TestDelegationToken.java
(original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/security/TestDelegationToken.java
Wed Feb 13 22:41:15 2013
@@ -76,8 +76,6 @@ public class TestDelegationToken {
   public void tearDown() throws Exception {
     LOG.info("starting shutting down the cluster");
     if(cluster!=null) {
-      cluster.getNameNode().getNamesystem().getDelegationTokenSecretManager()
-             .stopThreads();
       cluster.shutdown();
     }
     LOG.info("finished shutting down the cluster");



Mime
View raw message