hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject [2/7] git commit: HADOOP-11071. KMSClientProvider should drain the local generated EEK cache on key rollover. (tucu)
Date Mon, 08 Sep 2014 22:00:37 GMT
HADOOP-11071. KMSClientProvider should drain the local generated EEK cache on key rollover.
(tucu)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/df8c84cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/df8c84cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/df8c84cb

Branch: refs/heads/HDFS-6581
Commit: df8c84cba8512058f5097c6faeedf4b65cab3806
Parents: c1f8323
Author: Alejandro Abdelnur <tucu@apache.org>
Authored: Mon Sep 8 10:12:16 2014 -0700
Committer: Alejandro Abdelnur <tucu@apache.org>
Committed: Mon Sep 8 11:31:30 2014 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 +++
 .../crypto/key/KeyProviderCryptoExtension.java  | 11 ++++++++++
 .../crypto/key/kms/KMSClientProvider.java       |  9 +++++++-
 .../hadoop/crypto/key/kms/ValueQueue.java       | 13 ++++++++++++
 .../hadoop/crypto/key/TestValueQueue.java       | 14 +++++++++++++
 ...rKeyGeneratorKeyProviderCryptoExtension.java | 22 ++++++++++++++++++++
 .../hadoop/crypto/key/kms/server/TestKMS.java   | 17 +++++++++++++++
 7 files changed, 88 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/df8c84cb/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index fe011fd..0417b0a 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -771,6 +771,9 @@ Release 2.6.0 - UNRELEASED
     HADOOP-11073. Credential Provider related Unit Tests Failure on Windows.
     (Xiaoyu Yao via cnauroth)
 
+    HADOOP-11071. KMSClientProvider should drain the local generated EEK cache
+    on key rollover. (tucu)
+
 Release 2.5.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/df8c84cb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
index e2fb5cb..fed7e9e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
@@ -179,6 +179,13 @@ public class KeyProviderCryptoExtension extends
         throws IOException;
 
     /**
+     * Drains the Queue for the provided key.
+     *
+     * @param keyName the key to drain the Queue for
+     */
+    public void drain(String keyName);
+
+    /**
      * Generates a key material and encrypts it using the given key version name
      * and initialization vector. The generated key material is of the same
      * length as the <code>KeyVersion</code> material of the latest key version
@@ -313,6 +320,10 @@ public class KeyProviderCryptoExtension extends
       // NO-OP since the default version does not cache any keys
     }
 
+    @Override
+    public void drain(String keyName) {
+      // NO-OP since the default version does not cache any keys
+    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/df8c84cb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index acbe096..899b6c4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -590,7 +590,9 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
     conn.setRequestProperty(CONTENT_TYPE, APPLICATION_JSON_MIME);
     Map response = call(conn, jsonMaterial,
         HttpURLConnection.HTTP_OK, Map.class);
-    return parseJSONKeyVersion(response);
+    KeyVersion keyVersion = parseJSONKeyVersion(response);
+    encKeyVersionQueue.drain(name);
+    return keyVersion;
   }
 
 
@@ -713,6 +715,11 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
   }
 
   @Override
+  public void drain(String keyName) {
+    encKeyVersionQueue.drain(keyName);
+  }
+
+  @Override
   public Token<?>[] addDelegationTokens(String renewer,
       Credentials credentials) throws IOException {
     Token<?>[] tokens;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/df8c84cb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
index a415e2e..ee10483 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
@@ -228,6 +228,19 @@ public class ValueQueue <E> {
   }
 
   /**
+   * Drains the Queue for the provided key.
+   *
+   * @param keyName the key to drain the Queue for
+   */
+  public void drain(String keyName ) {
+    try {
+      keyQueues.get(keyName).clear();
+    } catch (ExecutionException ex) {
+      //NOP
+    }
+  }
+
+  /**
    * This removes the "num" values currently at the head of the Queue for the
    * provided key. Will immediately fire the Queue filler function if key
    * does not exist

http://git-wip-us.apache.org/repos/asf/hadoop/blob/df8c84cb/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java
index 7946588..8e3a093 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java
@@ -187,4 +187,18 @@ public class TestValueQueue {
     Assert.assertEquals(10, filler.getTop().num);
     vq.shutdown();
   }
+
+  @Test
+  public void testDrain() throws Exception {
+    MockFiller filler = new MockFiller();
+    ValueQueue<String> vq =
+        new ValueQueue<String>(10, 0.1f, 300, 1,
+            SyncGenerationPolicy.ALL, filler);
+    Assert.assertEquals("test", vq.getNext("k1"));
+    Assert.assertEquals(1, filler.getTop().num);
+    vq.drain("k1");
+    Assert.assertNull(filler.getTop());
+    vq.shutdown();
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/df8c84cb/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java
index a952cfe..a33f4f1 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.crypto.key.kms.server;
 
 import java.io.IOException;
 import java.security.GeneralSecurityException;
+import java.security.NoSuchAlgorithmException;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Queue;
@@ -27,6 +28,7 @@ import java.util.concurrent.ExecutionException;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
 import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
 import org.apache.hadoop.crypto.key.kms.ValueQueue;
 import org.apache.hadoop.crypto.key.kms.ValueQueue.SyncGenerationPolicy;
@@ -113,6 +115,11 @@ public class EagerKeyGeneratorKeyProviderCryptoExtension
     }
 
     @Override
+    public void drain(String keyName) {
+      encKeyVersionQueue.drain(keyName);
+    }
+
+    @Override
     public EncryptedKeyVersion generateEncryptedKey(String encryptionKeyName)
         throws IOException, GeneralSecurityException {
       try {
@@ -146,4 +153,19 @@ public class EagerKeyGeneratorKeyProviderCryptoExtension
         new CryptoExtension(conf, keyProviderCryptoExtension));
   }
 
+  @Override
+  public KeyVersion rollNewVersion(String name)
+      throws NoSuchAlgorithmException, IOException {
+    KeyVersion keyVersion = super.rollNewVersion(name);
+    getExtension().drain(name);
+    return keyVersion;
+  }
+
+  @Override
+  public KeyVersion rollNewVersion(String name, byte[] material)
+      throws IOException {
+    KeyVersion keyVersion = super.rollNewVersion(name, material);
+    getExtension().drain(name);
+    return keyVersion;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/df8c84cb/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index b921c84..74eab5c 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -531,6 +531,7 @@ public class TestKMS {
         Assert.assertEquals("d", meta.getDescription());
         Assert.assertEquals(attributes, meta.getAttributes());
 
+        // test delegation token retrieval
         KeyProviderDelegationTokenExtension kpdte =
             KeyProviderDelegationTokenExtension.
                 createKeyProviderDelegationTokenExtension(kp);
@@ -542,6 +543,22 @@ public class TestKMS {
 
         Assert.assertEquals(new Text("kms-dt"), credentials.getToken(
             SecurityUtil.buildTokenService(kmsAddr)).getKind());
+
+
+        // test rollover draining
+        KeyProviderCryptoExtension kpce = KeyProviderCryptoExtension.
+            createKeyProviderCryptoExtension(kp);
+        options = new KeyProvider.Options(conf);
+        options.setCipher("AES/CTR/NoPadding");
+        options.setBitLength(128);
+        kpce.createKey("k6", options);
+
+        EncryptedKeyVersion ekv1 = kpce.generateEncryptedKey("k6");
+        kpce.rollNewVersion("k6");
+        EncryptedKeyVersion ekv2 = kpce.generateEncryptedKey("k6");
+        Assert.assertNotEquals(ekv1.getEncryptionKeyVersionName(),
+            ekv2.getEncryptionKeyVersionName());
+
         return null;
       }
     });


Mime
View raw message