Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 08C7510CA8 for ; Sat, 2 Nov 2013 00:52:20 +0000 (UTC) Received: (qmail 33840 invoked by uid 500); 2 Nov 2013 00:52:19 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 33788 invoked by uid 500); 2 Nov 2013 00:52:19 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 33781 invoked by uid 99); 2 Nov 2013 00:52:19 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 02 Nov 2013 00:52:19 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 02 Nov 2013 00:52:16 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id D51F3238896F; Sat, 2 Nov 2013 00:51:54 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1538115 - in /hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/security/ src/main/resources/ Date: Sat, 02 Nov 2013 00:51:54 -0000 To: common-commits@hadoop.apache.org From: cmccabe@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20131102005154.D51F3238896F@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: cmccabe Date: Sat Nov 2 00:51:54 2013 New Revision: 1538115 URL: http://svn.apache.org/r1538115 Log: HADOOP-10079. log a warning message if group resolution takes too long (cmccabe) Modified: hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml Modified: hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1538115&r1=1538114&r2=1538115&view=diff ============================================================================== --- hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/CHANGES.txt Sat Nov 2 00:51:54 2013 @@ -11,6 +11,9 @@ Release 2.2.1 - UNRELEASED HADOOP-10046. Print a log message when SSL is enabled. (David S. Wang via wang) + HADOOP-10079. log a warning message if group resolution takes too long. + (cmccabe) + OPTIMIZATIONS BUG FIXES Modified: hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java?rev=1538115&r1=1538114&r2=1538115&view=diff ============================================================================== --- hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java (original) +++ hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java Sat Nov 2 00:51:54 2013 @@ -235,6 +235,14 @@ public class CommonConfigurationKeysPubl public static final String HADOOP_SECURITY_GROUPS_CACHE_SECS = "hadoop.security.groups.cache.secs"; /** See core-default.xml */ + public static final long HADOOP_SECURITY_GROUPS_CACHE_SECS_DEFAULT = + 300; + /** See core-default.xml */ + public static final String HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS = + "hadoop.security.groups.cache.warn.after.ms"; + public static final long HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS_DEFAULT = + 5000; + /** See core-default.xml */ public static final String HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authentication"; /** See core-default.xml */ Modified: hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java?rev=1538115&r1=1538114&r2=1538115&view=diff ============================================================================== --- hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java (original) +++ hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java Sat Nov 2 00:51:54 2013 @@ -50,6 +50,7 @@ public class Groups { private final Map userToGroupsMap = new ConcurrentHashMap(); private final long cacheTimeout; + private final long warningDeltaMs; public Groups(Configuration conf) { impl = @@ -60,11 +61,16 @@ public class Groups { conf); cacheTimeout = - conf.getLong(CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_SECS, 5*60) * 1000; + conf.getLong(CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_SECS, + CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_SECS_DEFAULT) * 1000; + warningDeltaMs = + conf.getLong(CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS, + CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS_DEFAULT); if(LOG.isDebugEnabled()) LOG.debug("Group mapping impl=" + impl.getClass().getName() + - "; cacheTimeout=" + cacheTimeout); + "; cacheTimeout=" + cacheTimeout + "; warningDeltaMs=" + + warningDeltaMs); } /** @@ -76,17 +82,24 @@ public class Groups { public List getGroups(String user) throws IOException { // Return cached value if available CachedGroups groups = userToGroupsMap.get(user); - long now = Time.now(); + long startMs = Time.monotonicNow(); // if cache has a value and it hasn't expired - if (groups != null && (groups.getTimestamp() + cacheTimeout > now)) { + if (groups != null && (groups.getTimestamp() + cacheTimeout > startMs)) { if(LOG.isDebugEnabled()) { LOG.debug("Returning cached groups for '" + user + "'"); } return groups.getGroups(); } - + // Create and cache user's groups - groups = new CachedGroups(impl.getGroups(user)); + List groupList = impl.getGroups(user); + long endMs = Time.monotonicNow(); + long deltaMs = endMs - startMs ; + if (deltaMs > warningDeltaMs) { + LOG.warn("Potential performance problem: getGroups(user=" + user +") " + + "took " + deltaMs + " milliseconds."); + } + groups = new CachedGroups(groupList, endMs); if (groups.getGroups().isEmpty()) { throw new IOException("No groups found for user " + user); } @@ -133,9 +146,9 @@ public class Groups { /** * Create and initialize group cache */ - CachedGroups(List groups) { + CachedGroups(List groups, long timestamp) { this.groups = groups; - this.timestamp = Time.now(); + this.timestamp = timestamp; } /** Modified: hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml?rev=1538115&r1=1538114&r2=1538115&view=diff ============================================================================== --- hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml (original) +++ hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml Sat Nov 2 00:51:54 2013 @@ -106,6 +106,15 @@ + hadoop.security.groups.cache.warn.after.ms + 5000 + + If looking up a single user to group takes longer than this amount of + milliseconds, we will log a warning message. + + + + hadoop.security.group.mapping.ldap.url