Return-Path: X-Original-To: apmail-argus-commits-archive@minotaur.apache.org Delivered-To: apmail-argus-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 30E3E1734E for ; Tue, 21 Oct 2014 03:04:19 +0000 (UTC) Received: (qmail 34474 invoked by uid 500); 21 Oct 2014 03:04:18 -0000 Delivered-To: apmail-argus-commits-archive@argus.apache.org Received: (qmail 34451 invoked by uid 500); 21 Oct 2014 03:04:18 -0000 Mailing-List: contact commits-help@argus.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@argus.incubator.apache.org Delivered-To: mailing list commits@argus.incubator.apache.org Received: (qmail 34441 invoked by uid 99); 21 Oct 2014 03:04:18 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 21 Oct 2014 03:04:18 +0000 X-ASF-Spam-Status: No, hits=-2001.4 required=5.0 tests=ALL_TRUSTED,RP_MATCHES_RCVD X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO mail.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with SMTP; Tue, 21 Oct 2014 03:03:55 +0000 Received: (qmail 27538 invoked by uid 99); 21 Oct 2014 03:02:38 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 21 Oct 2014 03:02:38 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 2C58893522C; Tue, 21 Oct 2014 03:02:37 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: dillidorai@apache.org To: commits@argus.incubator.apache.org Message-Id: <85556802ee864c869f15996f0f19b908@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: git commit: ARGUS-120-: argus ldap usersync can not import over 1000 users Date: Tue, 21 Oct 2014 03:02:37 +0000 (UTC) X-Virus-Checked: Checked by ClamAV on apache.org Repository: incubator-argus Updated Branches: refs/heads/master 48e3730f0 -> 662cd2b69 ARGUS-120-: argus ldap usersync can not import over 1000 users Project: http://git-wip-us.apache.org/repos/asf/incubator-argus/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-argus/commit/662cd2b6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-argus/tree/662cd2b6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-argus/diff/662cd2b6 Branch: refs/heads/master Commit: 662cd2b6935750668d0e27ce4e50c339ddc5be20 Parents: 48e3730 Author: Dilli Dorai Arumugam Authored: Fri Oct 17 07:08:56 2014 -0700 Committer: Dilli Dorai Arumugam Committed: Sat Oct 18 05:53:20 2014 -0700 ---------------------------------------------------------------------- .../process/LdapUserGroupBuilder.java | 141 ++++++++++++------- 1 file changed, 89 insertions(+), 52 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/662cd2b6/ugsync/src/main/java/com/xasecure/ldapusersync/process/LdapUserGroupBuilder.java ---------------------------------------------------------------------- diff --git a/ugsync/src/main/java/com/xasecure/ldapusersync/process/LdapUserGroupBuilder.java b/ugsync/src/main/java/com/xasecure/ldapusersync/process/LdapUserGroupBuilder.java index be29ca5..c8d3204 100644 --- a/ugsync/src/main/java/com/xasecure/ldapusersync/process/LdapUserGroupBuilder.java +++ b/ugsync/src/main/java/com/xasecure/ldapusersync/process/LdapUserGroupBuilder.java @@ -31,10 +31,13 @@ import javax.naming.Context; import javax.naming.InvalidNameException; import javax.naming.NamingEnumeration; import javax.naming.directory.Attribute; -import javax.naming.directory.DirContext; -import javax.naming.directory.InitialDirContext; import javax.naming.directory.SearchControls; import javax.naming.directory.SearchResult; +import javax.naming.ldap.Control; +import javax.naming.ldap.InitialLdapContext; +import javax.naming.ldap.LdapContext; +import javax.naming.ldap.PagedResultsControl; +import javax.naming.ldap.PagedResultsResponseControl; import org.apache.log4j.Logger; @@ -46,13 +49,15 @@ public class LdapUserGroupBuilder implements UserGroupSource { private static final Logger LOG = Logger.getLogger(LdapUserGroupBuilder.class); + private static final int PAGE_SIZE = 100; + private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); private String userSearchBase; private String extendedSearchFilter; private String userNameAttribute; - private DirContext dirContext; + private LdapContext ldapContext; private SearchControls searchControls; private boolean userNameCaseConversionFlag = false ; @@ -96,7 +101,7 @@ public class LdapUserGroupBuilder implements UserGroupSource { // do nothing } - private void createDirContext() throws Throwable { + private void createLdapContext() throws Throwable { LOG.info("LdapUserGroupBuilder initialization started"); String ldapUrl = config.getLdapUrl(); String ldapBindDn = config.getLdapBindDn(); @@ -113,7 +118,7 @@ public class LdapUserGroupBuilder implements UserGroupSource { env.put(Context.SECURITY_AUTHENTICATION, ldapAuthenticationMechanism); env.put(Context.REFERRAL, "follow") ; - dirContext = new InitialDirContext(env); + ldapContext = new InitialLdapContext(env, null); userSearchBase = config.getUserSearchBase(); int userSearchScope = config.getUserSearchScope(); @@ -160,9 +165,9 @@ public class LdapUserGroupBuilder implements UserGroupSource { } - private void closeDirContext() throws Throwable { - if (dirContext != null) { - dirContext.close(); + private void closeLdapContext() throws Throwable { + if (ldapContext != null) { + ldapContext.close(); } } @@ -176,75 +181,107 @@ public class LdapUserGroupBuilder implements UserGroupSource { public void updateSink(UserGroupSink sink) throws Throwable { LOG.info("LDAPUserGroupBuilder updateSink started"); try { - createDirContext(); + createLdapContext(); + + // Activate paged results + byte[] cookie = null; + ldapContext.setRequestControls(new Control[]{ + new PagedResultsControl(PAGE_SIZE, Control.NONCRITICAL) }); + int total; + int counter = 0; - NamingEnumeration searchResultEnum = dirContext + do { + NamingEnumeration searchResultEnum = ldapContext .search(userSearchBase, extendedSearchFilter, searchControls); - while (searchResultEnum.hasMore()) { - // searchResults contains all the user entries - final SearchResult userEntry = searchResultEnum.next(); - String userName = (String) userEntry.getAttributes() + while (searchResultEnum.hasMore()) { + // searchResults contains all the user entries + final SearchResult userEntry = searchResultEnum.next(); + String userName = (String) userEntry.getAttributes() .get(userNameAttribute).get(); - if (userNameCaseConversionFlag) { - if (userNameLowerCaseFlag) { - userName = userName.toLowerCase() ; - } - else { - userName = userName.toUpperCase() ; + if (userNameCaseConversionFlag) { + if (userNameLowerCaseFlag) { + userName = userName.toLowerCase() ; + } + else { + userName = userName.toUpperCase() ; + } } - } - Set groups = new HashSet(); - Set userGroupNameAttributeSet = config.getUserGroupNameAttributeSet(); - for (String useGroupNameAttribute : userGroupNameAttributeSet) { - Attribute userGroupfAttribute = userEntry.getAttributes().get(useGroupNameAttribute); - if(userGroupfAttribute != null) { - NamingEnumeration groupEnum = userGroupfAttribute.getAll(); - while (groupEnum.hasMore()) { - String gName = getShortGroupName((String) groupEnum + Set groups = new HashSet(); + Set userGroupNameAttributeSet = config.getUserGroupNameAttributeSet(); + for (String useGroupNameAttribute : userGroupNameAttributeSet) { + Attribute userGroupfAttribute = userEntry.getAttributes().get(useGroupNameAttribute); + if(userGroupfAttribute != null) { + NamingEnumeration groupEnum = userGroupfAttribute.getAll(); + while (groupEnum.hasMore()) { + String gName = getShortGroupName((String) groupEnum .next()); - if (groupNameCaseConversionFlag) { - if (groupNameLowerCaseFlag) { - gName = gName.toLowerCase(); - } else { - gName = gName.toUpperCase(); + if (groupNameCaseConversionFlag) { + if (groupNameLowerCaseFlag) { + gName = gName.toLowerCase(); + } else { + gName = gName.toUpperCase(); + } } + groups.add(gName); } - groups.add(gName); } } - } - List groupList = new ArrayList(groups); - counter++; - if (counter <= 1000) { - if (LOG.isInfoEnabled()) { - LOG.info("Updating user count: " + counter + List groupList = new ArrayList(groups); + counter++; + if (counter <= 2000) { + if (LOG.isInfoEnabled()) { + LOG.info("Updating user count: " + counter + ", userName: " + userName + ", groupList: " + groupList); - } - } else { - if (LOG.isTraceEnabled()) { - LOG.trace("Updating user count: " + counter + } + } else { + if (LOG.isTraceEnabled()) { + LOG.trace("Updating user count: " + counter + ", userName: " + userName + ", groupList: " + groupList); + } } - } - try { - sink.addOrUpdateUser(userName, groupList); - } catch (Throwable t) { - LOG.error("sink.addOrUpdateUser failed with exception: " + t.getMessage() + try { + sink.addOrUpdateUser(userName, groupList); + } catch (Throwable t) { + LOG.error("sink.addOrUpdateUser failed with exception: " + t.getMessage() + ", for user: " + userName + ", groups: " + groupList); + } } - } + + // Examine the paged results control response + Control[] controls = ldapContext.getResponseControls(); + if (controls != null) { + for (int i = 0; i < controls.length; i++) { + if (controls[i] instanceof PagedResultsResponseControl) { + PagedResultsResponseControl prrc = + (PagedResultsResponseControl)controls[i]; + total = prrc.getResultSize(); + if (total != 0) { + LOG.debug("END-OF-PAGE total : " + total); + } else { + LOG.debug("END-OF-PAGE total : unknown"); + } + cookie = prrc.getCookie(); + } + } + } else { + LOG.debug("No controls were sent from the server"); + } + // Re-activate paged results + ldapContext.setRequestControls(new Control[]{ + new PagedResultsControl(PAGE_SIZE, cookie, Control.CRITICAL) }); + } while (cookie != null); LOG.info("LDAPUserGroupBuilder.updateSink() completed with user count: " + counter); } finally { - closeDirContext(); + closeLdapContext(); } }