hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject [1/2] git commit: HBASE-10831 IntegrationTestIngestWithACL is not setting up LoadTestTool correctly (Vandana Ayyalasomayajula)
Date Fri, 23 May 2014 02:01:10 GMT
Repository: hbase
Updated Branches:
  refs/heads/0.98 c93ceed29 -> d89342e41
  refs/heads/master fd94fcde5 -> 41691e469


HBASE-10831 IntegrationTestIngestWithACL is not setting up LoadTestTool correctly (Vandana
Ayyalasomayajula)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/41691e46
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/41691e46
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/41691e46

Branch: refs/heads/master
Commit: 41691e469ae4f59aa04bfec99b1e8a19699f1d04
Parents: fd94fcd
Author: Andrew Purtell <apurtell@apache.org>
Authored: Thu May 22 18:56:20 2014 -0700
Committer: Andrew Purtell <apurtell@apache.org>
Committed: Thu May 22 18:56:20 2014 -0700

----------------------------------------------------------------------
 .../hbase/IntegrationTestIngestWithACL.java     | 32 ++++++++-
 .../apache/hadoop/hbase/util/LoadTestTool.java  | 76 +++++++++++++++++---
 .../hbase/util/MultiThreadedReaderWithACL.java  |  7 +-
 .../hbase/util/MultiThreadedUpdaterWithACL.java | 19 +++--
 4 files changed, 116 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/41691e46/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithACL.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithACL.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithACL.java
index 6b369f0..d635b98 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithACL.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithACL.java
@@ -22,6 +22,7 @@ import java.util.Arrays;
 import java.util.List;
 
 import org.apache.commons.cli.CommandLine;
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.security.access.AccessController;
@@ -45,8 +46,10 @@ public class IntegrationTestIngestWithACL extends IntegrationTestIngest
{
   private static final int SPECIAL_PERM_CELL_INSERTION_FACTOR = 100;
   public static final String OPT_SUPERUSER = "superuser";
   public static final String OPT_USERS = "userlist";
+  public static final String OPT_AUTHN = "authinfo";
   private String superUser = "owner";
   private String userNames = "user1,user2,user3,user4"; 
+  private String authnFileName;
   @Override
   public void setUpCluster() throws Exception {
     util = getTestingUtil(null);
@@ -66,6 +69,10 @@ public class IntegrationTestIngestWithACL extends IntegrationTestIngest
{
     tmp.add(HYPHEN + LoadTestTool.OPT_GENERATOR);
     StringBuilder sb = new StringBuilder(LoadTestDataGeneratorWithACL.class.getName());
     sb.append(COLON);
+    if (LoadTestTool.isSecure(getConf())) {
+      sb.append(authnFileName);
+      sb.append(COLON);
+    }
     sb.append(superUser);
     sb.append(COLON);
     sb.append(userNames);
@@ -80,7 +87,15 @@ public class IntegrationTestIngestWithACL extends IntegrationTestIngest
{
     super.addOptWithArg(OPT_SUPERUSER,
         "Super user name used to add the ACL permissions");
     super.addOptWithArg(OPT_USERS,
-        "List of users to be added with the ACLs.  Should be comma seperated.");
+      "List of users to be added with the ACLs.  Should be comma seperated.");
+    super
+        .addOptWithArg(
+          OPT_AUTHN,
+          "The name of the properties file that contains kerberos key tab file and principal
definitions. " +
+          "The principal key in the file should be of the form hbase.<username>.kerberos.principal."
+
+          " The keytab key in the file should be of the form hbase.<username>.keytab.file.
Example:  " +
+          "hbase.user1.kerberos.principal=user1/fully.qualified.domain.name@YOUR-REALM.COM,
" +
+          "hbase.user1.keytab.file=<filelocation>.");
   }
 
   @Override
@@ -92,6 +107,21 @@ public class IntegrationTestIngestWithACL extends IntegrationTestIngest
{
     if (cmd.hasOption(OPT_USERS)) {
       userNames = cmd.getOptionValue(OPT_USERS);
     }
+    if (LoadTestTool.isSecure(getConf())) {
+      boolean authFileNotFound = false;
+      if (cmd.hasOption(OPT_AUTHN)) {
+        authnFileName = cmd.getOptionValue(OPT_AUTHN);
+        if (StringUtils.isEmpty(authnFileName)) {
+          authFileNotFound = true;
+        }
+      } else {
+        authFileNotFound = true;
+      }
+      if (authFileNotFound) {
+        super.printUsage();
+        System.exit(EXIT_FAILURE);
+      }
+    }
   }
 
   public static void main(String[] args) throws Exception {

http://git-wip-us.apache.org/repos/asf/hbase/blob/41691e46/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
index da15707..dd32f3b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
@@ -19,10 +19,12 @@ package org.apache.hadoop.hbase.util;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.lang.reflect.Constructor;
+import java.net.InetAddress;
 import java.security.SecureRandom;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Properties;
 import java.util.Random;
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -31,6 +33,7 @@ import javax.crypto.spec.SecretKeySpec;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -50,6 +53,8 @@ import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.security.access.AccessControlClient;
 import org.apache.hadoop.hbase.util.test.LoadTestDataGenerator;
 import org.apache.hadoop.hbase.util.test.LoadTestDataGeneratorWithACL;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.ToolRunner;
 
 /**
@@ -184,6 +189,8 @@ public class LoadTestTool extends AbstractHBaseTool {
   private String superUser;
 
   private String userNames = "user1, user2, user3, user4";
+  //This file is used to read authentication information in secure clusters.
+  private String authnFileName;
 
   // TODO: refactor LoadTestToolImpl somewhere to make the usage from tests less bad,
   //       console tool itself should only be used from console.
@@ -472,14 +479,30 @@ public class LoadTestTool extends AbstractHBaseTool {
     if (cmd.hasOption(OPT_GENERATOR)) {
       String[] clazzAndArgs = cmd.getOptionValue(OPT_GENERATOR).split(COLON);
       dataGen = getLoadGeneratorInstance(clazzAndArgs[0]);
-      String args[];
+      String[] args;
       if (dataGen instanceof LoadTestDataGeneratorWithACL) {
         LOG.info("ACL is on");
-        superUser = clazzAndArgs[1];
-        userNames = clazzAndArgs[2];
-        args = Arrays.copyOfRange(clazzAndArgs, 1,
-            clazzAndArgs.length);
-        userOwner = User.createUserForTesting(conf, superUser, new String[0]);
+        if (isSecure(conf)) {
+          LOG.info("Security is on.");
+          authnFileName = clazzAndArgs[1];
+          superUser = clazzAndArgs[2];
+          userNames = clazzAndArgs[3];
+          args = Arrays.copyOfRange(clazzAndArgs, 2, clazzAndArgs.length);
+          Properties authConfig = new Properties();
+          authConfig.load(this.getClass().getClassLoader().getResourceAsStream(authnFileName));
+          try {
+            addAuthInfoToConf(authConfig, conf, superUser, userNames);
+          } catch (IOException exp) {
+            LOG.error(exp);
+            return EXIT_FAILURE;
+          }
+          userOwner = User.create(loginAndReturnUGI(conf, superUser));
+        } else {
+          superUser = clazzAndArgs[1];
+          userNames = clazzAndArgs[2];
+          args = Arrays.copyOfRange(clazzAndArgs, 1, clazzAndArgs.length);
+          userOwner = User.createUserForTesting(conf, superUser, new String[0]);
+        }
       } else {
         args = clazzAndArgs.length == 1 ? new String[0] : Arrays.copyOfRange(clazzAndArgs,
1,
             clazzAndArgs.length);
@@ -491,9 +514,7 @@ public class LoadTestTool extends AbstractHBaseTool {
           minColsPerKey, maxColsPerKey, COLUMN_FAMILY);
     }
 
-    if(userOwner != null) {
-      conf.set("hadoop.security.authorization", "false");
-      conf.set("hadoop.security.authentication", "simple");
+    if (userOwner != null) {
       LOG.info("Granting permission for the user " + userOwner.getShortName());
       AccessControlProtos.Permission.Action[] actions = {
           AccessControlProtos.Permission.Action.ADMIN,
@@ -736,4 +757,41 @@ public class LoadTestTool extends AbstractHBaseTool {
       }
     }
   }
+
+  private void addAuthInfoToConf(Properties authConfig, Configuration conf, String owner,
+      String userList) throws IOException {
+    List<String> users = Arrays.asList(userList.split(","));
+    users.add(owner);
+    for (String user : users) {
+      String keyTabFileConfKey = "hbase." + user + ".keytab.file";
+      String principalConfKey = "hbase." + user + ".kerberos.principal";
+      if (!authConfig.containsKey(keyTabFileConfKey) || !authConfig.containsKey(principalConfKey))
{
+        throw new IOException("Authentication configs missing for user : " + user);
+      }
+    }
+    for (String key : authConfig.stringPropertyNames()) {
+      conf.set(key, authConfig.getProperty(key));
+    }
+    LOG.debug("Added authentication properties to config successfully.");
+  }
+
+  public static UserGroupInformation loginAndReturnUGI(Configuration conf, String username)
+      throws IOException {
+    String hostname = InetAddress.getLocalHost().getHostName();
+    String keyTabFileConfKey = "hbase." + username + ".keytab.file";
+    String keyTabFileLocation = conf.get(keyTabFileConfKey);
+    String principalConfKey = "hbase." + username + ".kerberos.principal";
+    String principal = SecurityUtil.getServerPrincipal(conf.get(principalConfKey), hostname);
+    if (keyTabFileLocation == null || principal == null) {
+      LOG.warn("Principal or key tab file null for : " + principalConfKey + ", "
+          + keyTabFileConfKey);
+    }
+    UserGroupInformation ugi =
+        UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keyTabFileLocation);
+    return ugi;
+  }
+
+  public static boolean isSecure(Configuration conf) {
+    return ("kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication")));
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/41691e46/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java
index 0067003..9f57453 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java
@@ -119,8 +119,13 @@ public class MultiThreadedReaderWithACL extends MultiThreadedReader {
       if (userNames != null && userNames.length > 0) {
         int mod = ((int) keyToRead % userNames.length);
         User user;
+        UserGroupInformation realUserUgi;
         if(!users.containsKey(userNames[mod])) {
-          UserGroupInformation realUserUgi = UserGroupInformation.createRemoteUser(userNames[mod]);
+          if(LoadTestTool.isSecure(conf)) {
+            realUserUgi = LoadTestTool.loginAndReturnUGI(conf, userNames[mod]);
+          } else {
+            realUserUgi = UserGroupInformation.createRemoteUser(userNames[mod]);
+          }
           user = User.create(realUserUgi);
           users.put(userNames[mod], user);
         } else {

http://git-wip-us.apache.org/repos/asf/hbase/blob/41691e46/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java
index b4e24e5..8ea6d04 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java
@@ -133,14 +133,19 @@ public class MultiThreadedUpdaterWithACL extends MultiThreadedUpdater
{
       if (userNames != null && userNames.length > 0) {
         int mod = ((int) rowKeyBase % userNames.length);
         User user;
-        if(!users.containsKey(userNames[mod])) {
-          UserGroupInformation realUserUgi = UserGroupInformation.createRemoteUser(userNames[mod]);
-          user = User.create(realUserUgi);
-          users.put(userNames[mod], user);
-        } else {
-          user = users.get(userNames[mod]);
-        }
+        UserGroupInformation realUserUgi;
         try {
+          if (!users.containsKey(userNames[mod])) {
+            if (LoadTestTool.isSecure(conf)) {
+              realUserUgi = LoadTestTool.loginAndReturnUGI(conf, userNames[mod]);
+            } else {
+              realUserUgi = UserGroupInformation.createRemoteUser(userNames[mod]);
+            }
+            user = User.create(realUserUgi);
+            users.put(userNames[mod], user);
+          } else {
+            user = users.get(userNames[mod]);
+          }
           Result result = (Result) user.runAs(action);
           return result;
         } catch (Exception ie) {


Mime
View raw message