Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 20A81102DA for ; Fri, 23 May 2014 02:01:11 +0000 (UTC) Received: (qmail 20350 invoked by uid 500); 23 May 2014 02:01:11 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 20210 invoked by uid 500); 23 May 2014 02:01:11 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 20175 invoked by uid 99); 23 May 2014 02:01:10 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 23 May 2014 02:01:10 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 9EBA398872B; Fri, 23 May 2014 02:01:10 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: apurtell@apache.org To: commits@hbase.apache.org Date: Fri, 23 May 2014 02:01:10 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/2] git commit: HBASE-10831 IntegrationTestIngestWithACL is not setting up LoadTestTool correctly (Vandana Ayyalasomayajula) Repository: hbase Updated Branches: refs/heads/0.98 c93ceed29 -> d89342e41 refs/heads/master fd94fcde5 -> 41691e469 HBASE-10831 IntegrationTestIngestWithACL is not setting up LoadTestTool correctly (Vandana Ayyalasomayajula) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/41691e46 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/41691e46 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/41691e46 Branch: refs/heads/master Commit: 41691e469ae4f59aa04bfec99b1e8a19699f1d04 Parents: fd94fcd Author: Andrew Purtell Authored: Thu May 22 18:56:20 2014 -0700 Committer: Andrew Purtell Committed: Thu May 22 18:56:20 2014 -0700 ---------------------------------------------------------------------- .../hbase/IntegrationTestIngestWithACL.java | 32 ++++++++- .../apache/hadoop/hbase/util/LoadTestTool.java | 76 +++++++++++++++++--- .../hbase/util/MultiThreadedReaderWithACL.java | 7 +- .../hbase/util/MultiThreadedUpdaterWithACL.java | 19 +++-- 4 files changed, 116 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/41691e46/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithACL.java ---------------------------------------------------------------------- diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithACL.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithACL.java index 6b369f0..d635b98 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithACL.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithACL.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.List; import org.apache.commons.cli.CommandLine; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.security.access.AccessController; @@ -45,8 +46,10 @@ public class IntegrationTestIngestWithACL extends IntegrationTestIngest { private static final int SPECIAL_PERM_CELL_INSERTION_FACTOR = 100; public static final String OPT_SUPERUSER = "superuser"; public static final String OPT_USERS = "userlist"; + public static final String OPT_AUTHN = "authinfo"; private String superUser = "owner"; private String userNames = "user1,user2,user3,user4"; + private String authnFileName; @Override public void setUpCluster() throws Exception { util = getTestingUtil(null); @@ -66,6 +69,10 @@ public class IntegrationTestIngestWithACL extends IntegrationTestIngest { tmp.add(HYPHEN + LoadTestTool.OPT_GENERATOR); StringBuilder sb = new StringBuilder(LoadTestDataGeneratorWithACL.class.getName()); sb.append(COLON); + if (LoadTestTool.isSecure(getConf())) { + sb.append(authnFileName); + sb.append(COLON); + } sb.append(superUser); sb.append(COLON); sb.append(userNames); @@ -80,7 +87,15 @@ public class IntegrationTestIngestWithACL extends IntegrationTestIngest { super.addOptWithArg(OPT_SUPERUSER, "Super user name used to add the ACL permissions"); super.addOptWithArg(OPT_USERS, - "List of users to be added with the ACLs. Should be comma seperated."); + "List of users to be added with the ACLs. Should be comma seperated."); + super + .addOptWithArg( + OPT_AUTHN, + "The name of the properties file that contains kerberos key tab file and principal definitions. " + + "The principal key in the file should be of the form hbase..kerberos.principal." + + " The keytab key in the file should be of the form hbase..keytab.file. Example: " + + "hbase.user1.kerberos.principal=user1/fully.qualified.domain.name@YOUR-REALM.COM, " + + "hbase.user1.keytab.file=."); } @Override @@ -92,6 +107,21 @@ public class IntegrationTestIngestWithACL extends IntegrationTestIngest { if (cmd.hasOption(OPT_USERS)) { userNames = cmd.getOptionValue(OPT_USERS); } + if (LoadTestTool.isSecure(getConf())) { + boolean authFileNotFound = false; + if (cmd.hasOption(OPT_AUTHN)) { + authnFileName = cmd.getOptionValue(OPT_AUTHN); + if (StringUtils.isEmpty(authnFileName)) { + authFileNotFound = true; + } + } else { + authFileNotFound = true; + } + if (authFileNotFound) { + super.printUsage(); + System.exit(EXIT_FAILURE); + } + } } public static void main(String[] args) throws Exception { http://git-wip-us.apache.org/repos/asf/hbase/blob/41691e46/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java index da15707..dd32f3b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java @@ -19,10 +19,12 @@ package org.apache.hadoop.hbase.util; import java.io.IOException; import java.io.InterruptedIOException; import java.lang.reflect.Constructor; +import java.net.InetAddress; import java.security.SecureRandom; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Properties; import java.util.Random; import java.util.concurrent.atomic.AtomicReference; @@ -31,6 +33,7 @@ import javax.crypto.spec.SecretKeySpec; import org.apache.commons.cli.CommandLine; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; @@ -50,6 +53,8 @@ import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.access.AccessControlClient; import org.apache.hadoop.hbase.util.test.LoadTestDataGenerator; import org.apache.hadoop.hbase.util.test.LoadTestDataGeneratorWithACL; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ToolRunner; /** @@ -184,6 +189,8 @@ public class LoadTestTool extends AbstractHBaseTool { private String superUser; private String userNames = "user1, user2, user3, user4"; + //This file is used to read authentication information in secure clusters. + private String authnFileName; // TODO: refactor LoadTestToolImpl somewhere to make the usage from tests less bad, // console tool itself should only be used from console. @@ -472,14 +479,30 @@ public class LoadTestTool extends AbstractHBaseTool { if (cmd.hasOption(OPT_GENERATOR)) { String[] clazzAndArgs = cmd.getOptionValue(OPT_GENERATOR).split(COLON); dataGen = getLoadGeneratorInstance(clazzAndArgs[0]); - String args[]; + String[] args; if (dataGen instanceof LoadTestDataGeneratorWithACL) { LOG.info("ACL is on"); - superUser = clazzAndArgs[1]; - userNames = clazzAndArgs[2]; - args = Arrays.copyOfRange(clazzAndArgs, 1, - clazzAndArgs.length); - userOwner = User.createUserForTesting(conf, superUser, new String[0]); + if (isSecure(conf)) { + LOG.info("Security is on."); + authnFileName = clazzAndArgs[1]; + superUser = clazzAndArgs[2]; + userNames = clazzAndArgs[3]; + args = Arrays.copyOfRange(clazzAndArgs, 2, clazzAndArgs.length); + Properties authConfig = new Properties(); + authConfig.load(this.getClass().getClassLoader().getResourceAsStream(authnFileName)); + try { + addAuthInfoToConf(authConfig, conf, superUser, userNames); + } catch (IOException exp) { + LOG.error(exp); + return EXIT_FAILURE; + } + userOwner = User.create(loginAndReturnUGI(conf, superUser)); + } else { + superUser = clazzAndArgs[1]; + userNames = clazzAndArgs[2]; + args = Arrays.copyOfRange(clazzAndArgs, 1, clazzAndArgs.length); + userOwner = User.createUserForTesting(conf, superUser, new String[0]); + } } else { args = clazzAndArgs.length == 1 ? new String[0] : Arrays.copyOfRange(clazzAndArgs, 1, clazzAndArgs.length); @@ -491,9 +514,7 @@ public class LoadTestTool extends AbstractHBaseTool { minColsPerKey, maxColsPerKey, COLUMN_FAMILY); } - if(userOwner != null) { - conf.set("hadoop.security.authorization", "false"); - conf.set("hadoop.security.authentication", "simple"); + if (userOwner != null) { LOG.info("Granting permission for the user " + userOwner.getShortName()); AccessControlProtos.Permission.Action[] actions = { AccessControlProtos.Permission.Action.ADMIN, @@ -736,4 +757,41 @@ public class LoadTestTool extends AbstractHBaseTool { } } } + + private void addAuthInfoToConf(Properties authConfig, Configuration conf, String owner, + String userList) throws IOException { + List users = Arrays.asList(userList.split(",")); + users.add(owner); + for (String user : users) { + String keyTabFileConfKey = "hbase." + user + ".keytab.file"; + String principalConfKey = "hbase." + user + ".kerberos.principal"; + if (!authConfig.containsKey(keyTabFileConfKey) || !authConfig.containsKey(principalConfKey)) { + throw new IOException("Authentication configs missing for user : " + user); + } + } + for (String key : authConfig.stringPropertyNames()) { + conf.set(key, authConfig.getProperty(key)); + } + LOG.debug("Added authentication properties to config successfully."); + } + + public static UserGroupInformation loginAndReturnUGI(Configuration conf, String username) + throws IOException { + String hostname = InetAddress.getLocalHost().getHostName(); + String keyTabFileConfKey = "hbase." + username + ".keytab.file"; + String keyTabFileLocation = conf.get(keyTabFileConfKey); + String principalConfKey = "hbase." + username + ".kerberos.principal"; + String principal = SecurityUtil.getServerPrincipal(conf.get(principalConfKey), hostname); + if (keyTabFileLocation == null || principal == null) { + LOG.warn("Principal or key tab file null for : " + principalConfKey + ", " + + keyTabFileConfKey); + } + UserGroupInformation ugi = + UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keyTabFileLocation); + return ugi; + } + + public static boolean isSecure(Configuration conf) { + return ("kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication"))); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/41691e46/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java index 0067003..9f57453 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java @@ -119,8 +119,13 @@ public class MultiThreadedReaderWithACL extends MultiThreadedReader { if (userNames != null && userNames.length > 0) { int mod = ((int) keyToRead % userNames.length); User user; + UserGroupInformation realUserUgi; if(!users.containsKey(userNames[mod])) { - UserGroupInformation realUserUgi = UserGroupInformation.createRemoteUser(userNames[mod]); + if(LoadTestTool.isSecure(conf)) { + realUserUgi = LoadTestTool.loginAndReturnUGI(conf, userNames[mod]); + } else { + realUserUgi = UserGroupInformation.createRemoteUser(userNames[mod]); + } user = User.create(realUserUgi); users.put(userNames[mod], user); } else { http://git-wip-us.apache.org/repos/asf/hbase/blob/41691e46/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java index b4e24e5..8ea6d04 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java @@ -133,14 +133,19 @@ public class MultiThreadedUpdaterWithACL extends MultiThreadedUpdater { if (userNames != null && userNames.length > 0) { int mod = ((int) rowKeyBase % userNames.length); User user; - if(!users.containsKey(userNames[mod])) { - UserGroupInformation realUserUgi = UserGroupInformation.createRemoteUser(userNames[mod]); - user = User.create(realUserUgi); - users.put(userNames[mod], user); - } else { - user = users.get(userNames[mod]); - } + UserGroupInformation realUserUgi; try { + if (!users.containsKey(userNames[mod])) { + if (LoadTestTool.isSecure(conf)) { + realUserUgi = LoadTestTool.loginAndReturnUGI(conf, userNames[mod]); + } else { + realUserUgi = UserGroupInformation.createRemoteUser(userNames[mod]); + } + user = User.create(realUserUgi); + users.put(userNames[mod], user); + } else { + user = users.get(userNames[mod]); + } Result result = (Result) user.runAs(action); return result; } catch (Exception ie) {