hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From asur...@apache.org
Subject [44/50] [abbrv] hadoop git commit: HDFS-10899. Add functionality to re-encrypt EDEKs.
Date Thu, 24 Aug 2017 19:36:37 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionHandler.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionHandler.java
new file mode 100644
index 0000000..f0dd92c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionHandler.java
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileSystemTestHelper;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.KMSUtil;
+import org.apache.hadoop.util.StopWatch;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.mockito.Mockito;
+import org.mockito.internal.util.reflection.Whitebox;
+import org.slf4j.LoggerFactory;
+import org.slf4j.event.Level;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertTrue;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_HANDLER_RATIO_KEY;
+import static org.junit.Assert.fail;
+
+/**
+ * Test class for ReencryptionHandler.
+ */
+public class TestReencryptionHandler {
+
+  protected static final org.slf4j.Logger LOG =
+      LoggerFactory.getLogger(TestReencryptionHandler.class);
+
+  @Rule
+  public Timeout globalTimeout = new Timeout(180 * 1000);
+
+  @Before
+  public void setup() {
+    GenericTestUtils.setLogLevel(ReencryptionHandler.LOG, Level.TRACE);
+  }
+
+  private ReencryptionHandler mockReencryptionhandler(final Configuration conf)
+      throws IOException {
+    // mock stuff to create a mocked ReencryptionHandler
+    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
+        JavaKeyStoreProvider.SCHEME_NAME + "://file" + new Path(
+            new FileSystemTestHelper().getTestRootDir(), "test.jks").toUri());
+    final EncryptionZoneManager ezm = Mockito.mock(EncryptionZoneManager.class);
+    final KeyProvider kp = KMSUtil.createKeyProvider(conf,
+        CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH);
+    Mockito.when(ezm.getProvider()).thenReturn(
+        KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp));
+    return new ReencryptionHandler(ezm, conf);
+  }
+
+  @Test
+  public void testThrottle() throws Exception {
+    final Configuration conf = new Configuration();
+    conf.setDouble(DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_HANDLER_RATIO_KEY,
+        0.5);
+    final ReencryptionHandler rh = mockReencryptionhandler(conf);
+
+    // mock StopWatches so all = 30s, locked = 20s. With ratio = .5, throttle
+    // should wait for 30 * 0.5 - 20 = 5s.
+    final StopWatch mockAll = Mockito.mock(StopWatch.class);
+    Mockito.when(mockAll.now(TimeUnit.MILLISECONDS)).thenReturn((long) 30000);
+    Mockito.when(mockAll.reset()).thenReturn(mockAll);
+    final StopWatch mockLocked = Mockito.mock(StopWatch.class);
+    Mockito.when(mockLocked.now(TimeUnit.MILLISECONDS))
+        .thenReturn((long) 20000);
+    Mockito.when(mockLocked.reset()).thenReturn(mockLocked);
+    final BlockingQueue<Runnable> queue = new LinkedBlockingQueue<>();
+    Whitebox.setInternalState(rh, "throttleTimerAll", mockAll);
+    Whitebox.setInternalState(rh, "throttleTimerLocked", mockLocked);
+    Whitebox.setInternalState(rh, "taskQueue", queue);
+    final StopWatch sw = new StopWatch().start();
+    rh.throttle();
+    sw.stop();
+    assertTrue("should have throttled for at least 4 second",
+        sw.now(TimeUnit.MILLISECONDS) > 8000);
+    assertTrue("should have throttled for at most 6 second",
+        sw.now(TimeUnit.MILLISECONDS) < 12000);
+  }
+
+  @Test
+  public void testThrottleNoOp() throws Exception {
+    final Configuration conf = new Configuration();
+    conf.setDouble(DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_HANDLER_RATIO_KEY,
+        0.5);
+    final ReencryptionHandler rh = mockReencryptionhandler(conf);
+
+    // mock StopWatches so all = 30s, locked = 10s. With ratio = .5, throttle
+    // should not happen.
+    StopWatch mockAll = Mockito.mock(StopWatch.class);
+    Mockito.when(mockAll.now()).thenReturn(new Long(30000));
+    Mockito.when(mockAll.reset()).thenReturn(mockAll);
+    StopWatch mockLocked = Mockito.mock(StopWatch.class);
+    Mockito.when(mockLocked.now()).thenReturn(new Long(10000));
+    Mockito.when(mockLocked.reset()).thenReturn(mockLocked);
+    final BlockingQueue<Runnable> queue = new LinkedBlockingQueue<>();
+    Whitebox.setInternalState(rh, "throttleTimerAll", mockAll);
+    Whitebox.setInternalState(rh, "throttleTimerLocked", mockLocked);
+    Whitebox.setInternalState(rh, "taskQueue", queue);
+    final Map<Long, ReencryptionUpdater.ZoneSubmissionTracker>
+        submissions = new HashMap<>();
+    Whitebox.setInternalState(rh, "submissions", submissions);
+    StopWatch sw = new StopWatch().start();
+    rh.throttle();
+    sw.stop();
+    assertTrue("should not have throttled",
+        sw.now(TimeUnit.MILLISECONDS) < 1000);
+  }
+
+  @Test
+  public void testThrottleConfigs() throws Exception {
+    final Configuration conf = new Configuration();
+    conf.setDouble(DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_HANDLER_RATIO_KEY,
+        -1.0);
+    try {
+      mockReencryptionhandler(conf);
+      fail("Should not be able to init");
+    } catch (IllegalArgumentException e) {
+      GenericTestUtils.assertExceptionContains(" is not positive", e);
+    }
+
+    conf.setDouble(DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_HANDLER_RATIO_KEY,
+        0.0);
+    try {
+      mockReencryptionhandler(conf);
+      fail("Should not be able to init");
+    } catch (IllegalArgumentException e) {
+      GenericTestUtils.assertExceptionContains(" is not positive", e);
+    }
+  }
+
+  @Test
+  public void testThrottleAccumulatingTasks() throws Exception {
+    final Configuration conf = new Configuration();
+    final ReencryptionHandler rh = mockReencryptionhandler(conf);
+
+    // mock tasks piling up
+    final Map<Long, ReencryptionUpdater.ZoneSubmissionTracker>
+        submissions = new HashMap<>();
+    final ReencryptionUpdater.ZoneSubmissionTracker zst =
+        new ReencryptionUpdater.ZoneSubmissionTracker();
+    submissions.put(new Long(1), zst);
+    Future mock = Mockito.mock(Future.class);
+    for (int i = 0; i < Runtime.getRuntime().availableProcessors() * 3; ++i) {
+      zst.addTask(mock);
+    }
+
+    Thread removeTaskThread = new Thread() {
+      public void run() {
+        try {
+          Thread.sleep(3000);
+        } catch (InterruptedException ie) {
+          LOG.info("removeTaskThread interrupted.");
+          Thread.currentThread().interrupt();
+        }
+        zst.getTasks().clear();
+      }
+    };
+
+    Whitebox.setInternalState(rh, "submissions", submissions);
+    final StopWatch sw = new StopWatch().start();
+    removeTaskThread.start();
+    rh.throttle();
+    sw.stop();
+    assertTrue("should have throttled for at least 3 second",
+        sw.now(TimeUnit.MILLISECONDS) > 3000);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml
index f9bb29e..a6980aa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml
@@ -603,5 +603,85 @@
         </comparator>
       </comparators>
     </test>
+
+    <!--More thorough test cases for re-encryption are in TestEncryptionZones-->
+    <test>
+      <description>Test success of reencrypt submission on a EZ</description>
+      <test-commands>
+        <command>-fs NAMENODE -mkdir /src</command>
+        <crypto-admin-command>-createZone -path /src -keyName myKey</crypto-admin-command>
+        <crypto-admin-command>-reencryptZone -start -path /src</crypto-admin-command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm -r -skipTrash /src</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>SubstringComparator</type>
+          <expected-output>successfully submitted for zone: /src action: START</expected-output>
+        </comparator>
+      </comparators>
+    </test>
+
+    <test>
+      <description>Test failure of reencrypt submission on a non-EZ</description>
+      <test-commands>
+        <command>-fs NAMENODE -mkdir /src</command>
+        <crypto-admin-command>-reencryptZone -start -path /src</crypto-admin-command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm -r -skipTrash /src</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>SubstringComparator</type>
+          <expected-output>not the root of an encryption zone</expected-output>
+        </comparator>
+      </comparators>
+    </test>
+
+    <!-- Cannot test successful cancel here, since the submit will finish very quickly.
+         TestReencryption covers successful cancellations. -->
+
+    <test>
+      <description>Test failure of reencrypt on cancellation a not-being-re-encrypted
EZ</description>
+      <test-commands>
+        <command>-fs NAMENODE -mkdir /src</command>
+        <crypto-admin-command>-createZone -path /src -keyName myKey</crypto-admin-command>
+        <crypto-admin-command>-reencryptZone -cancel -path /src</crypto-admin-command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm -r -skipTrash /src</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>SubstringComparator</type>
+          <expected-output>Zone /src is not under re-encryption</expected-output>
+        </comparator>
+      </comparators>
+    </test>
+
+    <test>
+      <description>Test success of list reencrypt status on a EZ</description>
+      <test-commands>
+        <command>-fs NAMENODE -mkdir /src</command>
+        <command>-fs NAMENODE -mkdir /zone2</command>
+        <crypto-admin-command>-createZone -path /src -keyName myKey</crypto-admin-command>
+        <crypto-admin-command>-createZone -path /zone2 -keyName myKey</crypto-admin-command>
+        <crypto-admin-command>-reencryptZone -start -path /src</crypto-admin-command>
+        <crypto-admin-command>-reencryptZone -start -path /zone2</crypto-admin-command>
+        <crypto-admin-command>-listReencryptionStatus</crypto-admin-command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm -r -skipTrash /src</command>
+        <command>-fs NAMENODE -rm -r -skipTrash /zone2</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>TokenComparator</type>
+          <expected-output>/src,Completed,false,myKey,zone2</expected-output>
+        </comparator>
+      </comparators>
+    </test>
   </tests>
 </configuration>


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message