hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ecl...@apache.org
Subject [09/50] [abbrv] hbase git commit: HBASE-16160 Support RPC encryption with direct ByteBuffers (Colin Ma via garyh)
Date Mon, 11 Jul 2016 23:48:02 GMT
HBASE-16160 Support RPC encryption with direct ByteBuffers (Colin Ma via garyh)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3b5fbf8d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3b5fbf8d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3b5fbf8d

Branch: refs/heads/HBASE-14850
Commit: 3b5fbf8d73b0ee50a0b04b044f93c1af39e5d79d
Parents: 356c130
Author: Gary Helmling <garyh@apache.org>
Authored: Mon Jul 11 10:33:22 2016 -0700
Committer: Gary Helmling <garyh@apache.org>
Committed: Mon Jul 11 10:33:22 2016 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hbase/ipc/BufferChain.java    |   5 +-
 .../hbase/security/token/SecureTestCluster.java | 123 +++++++++++++++++++
 .../TestDelegationTokenWithEncryption.java      |  91 ++++++++++++++
 .../token/TestGenerateDelegationToken.java      |  93 +-------------
 4 files changed, 218 insertions(+), 94 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/3b5fbf8d/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
index 39efa40..7adc94d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
@@ -63,8 +63,9 @@ class BufferChain {
     byte [] bytes = new byte [this.remaining];
     int offset = 0;
     for (ByteBuffer bb: this.buffers) {
-      System.arraycopy(bb.array(), bb.arrayOffset(), bytes, offset, bb.limit());
-      offset += bb.capacity();
+      int length = bb.remaining();
+      bb.get(bytes, offset, length);
+      offset += length;
     }
     return bytes;
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/3b5fbf8d/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
new file mode 100644
index 0000000..a469537
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.security.token;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.LocalHBaseCluster;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.File;
+import java.util.Properties;
+
+/**
+ * The class for set up a security cluster with kerberos, hdfs, hbase.
+ */
+public class SecureTestCluster {
+  protected static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  protected static String USERNAME;
+
+  private static LocalHBaseCluster CLUSTER;
+
+  private static final File KEYTAB_FILE = new File(TEST_UTIL.getDataTestDir("keytab").toUri()
+      .getPath());
+  private static MiniKdc KDC;
+
+  private static String HOST = "localhost";
+
+  private static String PRINCIPAL;
+
+  private static String HTTP_PRINCIPAL;
+
+  /**
+   * Setup the security configuration for hdfs.
+   */
+  private static void setHdfsSecuredConfiguration(Configuration conf) throws Exception {
+    // change XXX_USER_NAME_KEY to XXX_KERBEROS_PRINCIPAL_KEY after we drop support for hadoop-2.4.1
+    conf.set(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, PRINCIPAL + "@" + KDC.getRealm());
+    conf.set(DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY, KEYTAB_FILE.getAbsolutePath());
+    conf.set(DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY, PRINCIPAL + "@" + KDC.getRealm());
+    conf.set(DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY, KEYTAB_FILE.getAbsolutePath());
+    conf.set(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, HTTP_PRINCIPAL
+ "@"
+        + KDC.getRealm());
+    conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
+    conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name());
+    conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0");
+    conf.set(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0");
+
+    File keystoresDir = new File(TEST_UTIL.getDataTestDir("keystore").toUri().getPath());
+    keystoresDir.mkdirs();
+    String sslConfDir = KeyStoreTestUtil.getClasspathDir(TestGenerateDelegationToken.class);
+    KeyStoreTestUtil.setupSSLConfig(keystoresDir.getAbsolutePath(), sslConfDir, conf, false);
+
+    conf.setBoolean("ignore.secure.ports.for.testing", true);
+  }
+
+  /**
+   * Setup and start kerberos, hbase
+   */
+  @BeforeClass
+  public static void setUp() throws Exception {
+    Properties conf = MiniKdc.createConf();
+    conf.put(MiniKdc.DEBUG, true);
+    KDC = new MiniKdc(conf, new File(TEST_UTIL.getDataTestDir("kdc").toUri().getPath()));
+    KDC.start();
+    USERNAME = UserGroupInformation.getLoginUser().getShortUserName();
+    PRINCIPAL = USERNAME + "/" + HOST;
+    HTTP_PRINCIPAL = "HTTP/" + HOST;
+    KDC.createPrincipal(KEYTAB_FILE, PRINCIPAL, HTTP_PRINCIPAL);
+    TEST_UTIL.startMiniZKCluster();
+
+    HBaseKerberosUtils.setKeytabFileForTesting(KEYTAB_FILE.getAbsolutePath());
+    HBaseKerberosUtils.setPrincipalForTesting(PRINCIPAL + "@" + KDC.getRealm());
+    HBaseKerberosUtils.setSecuredConfiguration(TEST_UTIL.getConfiguration());
+    setHdfsSecuredConfiguration(TEST_UTIL.getConfiguration());
+    UserGroupInformation.setConfiguration(TEST_UTIL.getConfiguration());
+    TEST_UTIL.getConfiguration().setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
+        TokenProvider.class.getName());
+    TEST_UTIL.startMiniDFSCluster(1);
+    Path rootdir = TEST_UTIL.getDataTestDirOnTestFS("TestGenerateDelegationToken");
+    FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootdir);
+    CLUSTER = new LocalHBaseCluster(TEST_UTIL.getConfiguration(), 1);
+    CLUSTER.startup();
+  }
+
+  @AfterClass
+  public static void tearDown() throws Exception {
+    if (CLUSTER != null) {
+      CLUSTER.shutdown();
+    }
+    CLUSTER.join();
+    if (KDC != null) {
+      KDC.stop();
+    }
+    TEST_UTIL.shutdownMiniCluster();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/3b5fbf8d/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestDelegationTokenWithEncryption.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestDelegationTokenWithEncryption.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestDelegationTokenWithEncryption.java
new file mode 100644
index 0000000..60d4a6a
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestDelegationTokenWithEncryption.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.security.token;
+
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.ipc.AsyncRpcClient;
+import org.apache.hadoop.hbase.ipc.RpcClient;
+import org.apache.hadoop.hbase.ipc.RpcClientFactory;
+import org.apache.hadoop.hbase.ipc.RpcClientImpl;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.SecurityTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import java.io.IOException;
+
+@Category({ SecurityTests.class, MediumTests.class })
+public class TestDelegationTokenWithEncryption extends SecureTestCluster {
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+    // enable rpc encryption
+    TEST_UTIL.getConfiguration().set("hbase.rpc.protection", "privacy");
+    SecureTestCluster.setUp();
+  }
+
+  private void testPutGetWithDelegationToken(Class<? extends RpcClient> rpcImplClass)
+      throws IOException, ServiceException {
+    TEST_UTIL.getConfiguration().set(RpcClientFactory.CUSTOM_RPC_CLIENT_IMPL_CONF_KEY,
+        rpcImplClass.getName());
+    try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
+         Table table = conn.getTable(TableName.valueOf("testtable"));) {
+      Put p = new Put(Bytes.toBytes("row"));
+      p.addColumn(Bytes.toBytes("family"),
+          Bytes.toBytes("data"), Bytes.toBytes("testdata"));
+      table.put(p);
+      Get g = new Get(Bytes.toBytes("row"));
+      Result result = table.get(g);
+      Assert.assertArrayEquals(Bytes.toBytes("testdata"),
+          result.getValue(Bytes.toBytes("family"), Bytes.toBytes("data")));
+    }
+  }
+
+  @Test
+  public void testPutGetWithDelegationToken()  throws Exception {
+    try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()))
{
+      Token<? extends TokenIdentifier> token = TokenUtil.obtainToken(conn);
+      UserGroupInformation.getCurrentUser().addToken(token);
+      // create the table for test
+      Admin admin = conn.getAdmin();
+      HTableDescriptor tableDescriptor = new
+          HTableDescriptor(new HTableDescriptor(TableName.valueOf("testtable")));
+      tableDescriptor.addFamily(new HColumnDescriptor("family"));
+      admin.createTable(tableDescriptor);
+
+      testPutGetWithDelegationToken(RpcClientImpl.class);
+      testPutGetWithDelegationToken(AsyncRpcClient.class);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/3b5fbf8d/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestGenerateDelegationToken.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestGenerateDelegationToken.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestGenerateDelegationToken.java
index e3d3c27..1d7c676 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestGenerateDelegationToken.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestGenerateDelegationToken.java
@@ -20,21 +20,13 @@ package org.apache.hadoop.hbase.security.token;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
-import java.io.File;
 import java.io.IOException;
-import java.util.Properties;
 
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.LocalHBaseCluster;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.ConnectionFactory;
 import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
-import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil;
 import org.apache.hadoop.hbase.ipc.AsyncRpcClient;
 import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
 import org.apache.hadoop.hbase.ipc.RpcClient;
@@ -46,101 +38,19 @@ import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos.GetAuthen
 import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos.WhoAmIRequest;
 import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos.WhoAmIResponse;
 import org.apache.hadoop.hbase.security.AccessDeniedException;
-import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.testclassification.SecurityTests;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.minikdc.MiniKdc;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
 import com.google.protobuf.ServiceException;
 
 @Category({ SecurityTests.class, MediumTests.class })
-public class TestGenerateDelegationToken {
-
-  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-
-  private static LocalHBaseCluster CLUSTER;
-
-  private static final File KEYTAB_FILE = new File(TEST_UTIL.getDataTestDir("keytab").toUri()
-      .getPath());
-  private static MiniKdc KDC;
-
-  private static String HOST = "localhost";
-
-  private static String USERNAME;
-
-  private static String PRINCIPAL;
-
-  private static String HTTP_PRINCIPAL;
-
-  private static void setHdfsSecuredConfiguration(Configuration conf) throws Exception {
-    // change XXX_USER_NAME_KEY to XXX_KERBEROS_PRINCIPAL_KEY after we drop support for hadoop-2.4.1
-    conf.set(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, PRINCIPAL + "@" + KDC.getRealm());
-    conf.set(DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY, KEYTAB_FILE.getAbsolutePath());
-    conf.set(DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY, PRINCIPAL + "@" + KDC.getRealm());
-    conf.set(DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY, KEYTAB_FILE.getAbsolutePath());
-    conf.set(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, HTTP_PRINCIPAL
+ "@"
-        + KDC.getRealm());
-    conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
-    conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name());
-    conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0");
-    conf.set(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0");
-
-    File keystoresDir = new File(TEST_UTIL.getDataTestDir("keystore").toUri().getPath());
-    keystoresDir.mkdirs();
-    String sslConfDir = KeyStoreTestUtil.getClasspathDir(TestGenerateDelegationToken.class);
-    KeyStoreTestUtil.setupSSLConfig(keystoresDir.getAbsolutePath(), sslConfDir, conf, false);
-
-    conf.setBoolean("ignore.secure.ports.for.testing", true);
-  }
-
-  @BeforeClass
-  public static void setUp() throws Exception {
-    Properties conf = MiniKdc.createConf();
-    conf.put(MiniKdc.DEBUG, true);
-    KDC = new MiniKdc(conf, new File(TEST_UTIL.getDataTestDir("kdc").toUri().getPath()));
-    KDC.start();
-    USERNAME = UserGroupInformation.getLoginUser().getShortUserName();
-    PRINCIPAL = USERNAME + "/" + HOST;
-    HTTP_PRINCIPAL = "HTTP/" + HOST;
-    KDC.createPrincipal(KEYTAB_FILE, PRINCIPAL, HTTP_PRINCIPAL);
-    TEST_UTIL.startMiniZKCluster();
-
-    HBaseKerberosUtils.setKeytabFileForTesting(KEYTAB_FILE.getAbsolutePath());
-    HBaseKerberosUtils.setPrincipalForTesting(PRINCIPAL + "@" + KDC.getRealm());
-    HBaseKerberosUtils.setSecuredConfiguration(TEST_UTIL.getConfiguration());
-    setHdfsSecuredConfiguration(TEST_UTIL.getConfiguration());
-    UserGroupInformation.setConfiguration(TEST_UTIL.getConfiguration());
-    TEST_UTIL.getConfiguration().setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
-      TokenProvider.class.getName());
-    TEST_UTIL.startMiniDFSCluster(1);
-    Path rootdir = TEST_UTIL.getDataTestDirOnTestFS("TestGenerateDelegationToken");
-    FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootdir);
-    CLUSTER = new LocalHBaseCluster(TEST_UTIL.getConfiguration(), 1);
-    CLUSTER.startup();
-  }
-
-  @AfterClass
-  public static void tearDown() throws Exception {
-    if (CLUSTER != null) {
-      CLUSTER.shutdown();
-    }
-    CLUSTER.join();
-    if (KDC != null) {
-      KDC.stop();
-    }
-    TEST_UTIL.shutdownMiniCluster();
-  }
+public class TestGenerateDelegationToken extends SecureTestCluster {
 
   private void testTokenAuth(Class<? extends RpcClient> rpcImplClass) throws IOException,
       ServiceException {
@@ -172,6 +82,5 @@ public class TestGenerateDelegationToken {
       testTokenAuth(RpcClientImpl.class);
       testTokenAuth(AsyncRpcClient.class);
     }
-
   }
 }


Mime
View raw message