hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject [34/50] [abbrv] git commit: HADOOP-11161. Expose close method in KeyProvider to give clients of Provider implementations a hook to release resources. Contribued by Arun Suresh. (cherry picked from commit d9556e873ef4d3e68c4f0c991f856d1faa747f07) (cherry
Date Fri, 10 Oct 2014 19:01:13 GMT
HADOOP-11161. Expose close method in KeyProvider to give clients of Provider implementations
a hook to release resources. Contribued by Arun Suresh.
(cherry picked from commit d9556e873ef4d3e68c4f0c991f856d1faa747f07)
(cherry picked from commit 3a2565c7be80cf6e9cdfec0f5460ed8ed2252768)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e5503b8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e5503b8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e5503b8

Branch: refs/heads/champlain
Commit: 5e5503b8d93b8413ba2268d7058575b1665a7db4
Parents: 3cbe727
Author: Aaron T. Myers <atm@apache.org>
Authored: Wed Oct 8 17:58:53 2014 -0700
Committer: Jitendra Pandey <Jitendra@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Thu Oct 9 00:30:12 2014 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt   |  3 +++
 .../org/apache/hadoop/crypto/key/KeyProvider.java |  8 ++++++++
 .../crypto/key/KeyProviderCryptoExtension.java    |  7 +++++++
 .../hadoop/crypto/key/kms/KMSClientProvider.java  | 11 +++++++++++
 .../apache/hadoop/crypto/key/kms/ValueQueue.java  | 14 +++++++++++---
 .../java/org/apache/hadoop/hdfs/DFSClient.java    | 18 ++++++++++++------
 6 files changed, 52 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e5503b8/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index db2795c..69b7622 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -469,6 +469,9 @@ Release 2.6.0 - UNRELEASED
     HADOOP-10404. Some accesses to DomainSocketWatcher#closed are not protected
     by lock (cmccabe)
 
+    HADOOP-11161. Expose close method in KeyProvider to give clients of
+    Provider implementations a hook to release resources. (Arun Suresh via atm)
+
     BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
   
       HADOOP-10734. Implement high-performance secure random number sources.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e5503b8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
index a8b9414..9dd1d47 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
@@ -534,6 +534,14 @@ public abstract class KeyProvider {
                                             ) throws IOException;
 
   /**
+   * Can be used by implementing classes to close any resources
+   * that require closing
+   */
+  public void close() throws IOException {
+    // NOP
+  }
+
+  /**
    * Roll a new version of the given key generating the material for it.
    * <p/>
    * This implementation generates the key material and calls the

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e5503b8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
index f800689..73c9885 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
@@ -410,4 +410,11 @@ public class KeyProviderCryptoExtension extends
     return new KeyProviderCryptoExtension(keyProvider, cryptoExtension);
   }
 
+  @Override
+  public void close() throws IOException {
+    if (getKeyProvider() != null) {
+      getKeyProvider().close();
+    }
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e5503b8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 5b7f109..c4c7e0c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -791,4 +791,15 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
     return tokens;
   }
 
+  /**
+   * Shutdown valueQueue executor threads
+   */
+  @Override
+  public void close() throws IOException {
+    try {
+      encKeyVersionQueue.shutdown();
+    } catch (Exception e) {
+      throw new IOException(e);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e5503b8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
index ee10483..aa0e624 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
@@ -75,6 +75,8 @@ public class ValueQueue <E> {
   private final int numValues;
   private final float lowWatermark;
 
+  private volatile boolean executorThreadsStarted = false;
+
   /**
    * A <code>Runnable</code> which takes a string name.
    */
@@ -187,9 +189,6 @@ public class ValueQueue <E> {
             TimeUnit.MILLISECONDS, queue, new ThreadFactoryBuilder()
                 .setDaemon(true)
                 .setNameFormat(REFILL_THREAD).build());
-    // To ensure all requests are first queued, make coreThreads = maxThreads
-    // and pre-start all the Core Threads.
-    executor.prestartAllCoreThreads();
   }
 
   public ValueQueue(final int numValues, final float lowWaterMark, long expiry,
@@ -297,6 +296,15 @@ public class ValueQueue <E> {
 
   private void submitRefillTask(final String keyName,
       final Queue<E> keyQueue) throws InterruptedException {
+    if (!executorThreadsStarted) {
+      synchronized (this) {
+        // To ensure all requests are first queued, make coreThreads =
+        // maxThreads
+        // and pre-start all the Core Threads.
+        executor.prestartAllCoreThreads();
+        executorThreadsStarted = true;
+      }
+    }
     // The submit/execute method of the ThreadPoolExecutor is bypassed and
     // the Runnable is directly put in the backing BlockingQueue so that we
     // can control exactly how the runnable is inserted into the queue.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e5503b8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 66a4d0c..c26e3d1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -920,12 +920,18 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
    */
   @Override
   public synchronized void close() throws IOException {
-    if(clientRunning) {
-      closeAllFilesBeingWritten(false);
-      clientRunning = false;
-      getLeaseRenewer().closeClient(this);
-      // close connections to the namenode
-      closeConnectionToNamenode();
+    try {
+      if(clientRunning) {
+        closeAllFilesBeingWritten(false);
+        clientRunning = false;
+        getLeaseRenewer().closeClient(this);
+        // close connections to the namenode
+        closeConnectionToNamenode();
+      }
+    } finally {
+      if (provider != null) {
+        provider.close();
+      }
     }
   }
 


Mime
View raw message