ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rle...@apache.org
Subject ambari git commit: AMBARI-10513. ambari-server sync-ldap fails if there are too many users in the LDAP server (more than 1000?) (Emil Anca via rlevas)
Date Tue, 21 Apr 2015 01:56:10 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk c79bd1245 -> eeea2eebe


AMBARI-10513. ambari-server sync-ldap fails if there are too many users in the LDAP server
(more than 1000?) (Emil Anca via rlevas)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/eeea2eeb
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/eeea2eeb
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/eeea2eeb

Branch: refs/heads/trunk
Commit: eeea2eebec7d77ce4cea242b456c44f10610b331
Parents: c79bd12
Author: Emil Anca <eanca@hortonworks.com>
Authored: Mon Apr 20 21:55:56 2015 -0400
Committer: Robert Levas <rlevas@hortonworks.com>
Committed: Mon Apr 20 21:56:05 2015 -0400

----------------------------------------------------------------------
 .../security/ldap/AmbariLdapDataPopulator.java  | 33 +++++++++++++++++---
 .../ldap/AmbariLdapDataPopulatorTest.java       | 26 +++++++++++++--
 2 files changed, 51 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/eeea2eeb/ambari-server/src/main/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulator.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulator.java
b/ambari-server/src/main/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulator.java
index d1293cb..ada4171 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulator.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulator.java
@@ -28,6 +28,7 @@ import java.util.Set;
 
 import javax.naming.NamingException;
 import javax.naming.directory.Attributes;
+import javax.naming.directory.SearchControls;
 
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.configuration.Configuration;
@@ -37,6 +38,7 @@ import org.apache.ambari.server.security.authorization.User;
 import org.apache.ambari.server.security.authorization.Users;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.springframework.ldap.control.PagedResultsDirContextProcessor;
 import org.springframework.ldap.core.AttributesMapper;
 import org.springframework.ldap.core.ContextMapper;
 import org.springframework.ldap.core.DirContextAdapter;
@@ -83,6 +85,7 @@ public class AmbariLdapDataPopulator {
   // Constants
   private static final String UID_ATTRIBUTE          = "uid";
   private static final String OBJECT_CLASS_ATTRIBUTE = "objectClass";
+  private static final int USERS_PAGE_SIZE = 500;
 
   /**
    * Construct an AmbariLdapDataPopulator.
@@ -515,12 +518,19 @@ public class AmbariLdapDataPopulator {
     final Set<LdapUserDto> users = new HashSet<LdapUserDto>();
     final LdapTemplate ldapTemplate = loadLdapTemplate();
     String baseDn = ldapServerProperties.getBaseDN();
-
-    for (Object dto: ldapTemplate.search(baseDn, filter.encode(), new LdapUserContextMapper(ldapServerProperties)))
{
-      if (dto != null ) {
-        users.add((LdapUserDto) dto);
+    PagedResultsDirContextProcessor processor = createPagingProcessor();
+    SearchControls searchControls = new SearchControls();
+    searchControls.setSearchScope(SearchControls.SUBTREE_SCOPE);
+    LdapUserContextMapper ldapUserContextMapper = new LdapUserContextMapper(ldapServerProperties);
+    String encodedFilter = filter.encode();
+    
+    do {
+      for (Object dto : ldapTemplate.search(baseDn, encodedFilter, searchControls, ldapUserContextMapper,
processor)) {
+        if (dto != null) {
+          users.add((LdapUserDto)dto);
+        }
       }
-    }
+    } while (processor.getCookie().getCookie() != null);
     return users;
   }
 
@@ -583,6 +593,11 @@ public class AmbariLdapDataPopulator {
       ldapServerProperties = properties;
 
       final LdapContextSource ldapContextSource = createLdapContextSource();
+	  
+      // The LdapTemplate by design will close the connection after each call to the LDAP
Server
+      // In order to have the interaction work with large/paged results, said connection
must be pooled and reused
+      ldapContextSource.setPooled(true);
+
       final List<String> ldapUrls = ldapServerProperties.getLdapUrls();
       ldapContextSource.setUrls(ldapUrls.toArray(new String[ldapUrls.size()]));
 
@@ -617,6 +632,14 @@ public class AmbariLdapDataPopulator {
   }
 
   /**
+   * PagedResultsDirContextProcessor factory method.
+   * @return new processor;
+   */
+  protected PagedResultsDirContextProcessor createPagingProcessor() {
+    return new PagedResultsDirContextProcessor(USERS_PAGE_SIZE, null);
+  }
+
+  /**
    * LdapTemplate factory method.
    *
    * @param ldapContextSource  the LDAP context source

http://git-wip-us.apache.org/repos/asf/ambari/blob/eeea2eeb/ambari-server/src/test/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulatorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulatorTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulatorTest.java
index 09a2256..4968730 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulatorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulatorTest.java
@@ -43,12 +43,16 @@ import org.easymock.Capture;
 import org.easymock.EasyMock;
 import org.easymock.IAnswer;
 import org.junit.Test;
+import org.springframework.ldap.control.PagedResultsCookie;
+import org.springframework.ldap.control.PagedResultsDirContextProcessor;
 import org.springframework.ldap.core.AttributesMapper;
 import org.springframework.ldap.core.ContextMapper;
 import org.springframework.ldap.core.DirContextAdapter;
 import org.springframework.ldap.core.LdapTemplate;
 import org.springframework.ldap.core.support.LdapContextSource;
 
+import javax.naming.directory.SearchControls;
+
 import static junit.framework.Assert.*;
 import static org.easymock.EasyMock.*;
 import static org.easymock.EasyMock.createNiceMock;
@@ -69,6 +73,7 @@ public class AmbariLdapDataPopulatorTest {
 
     protected LdapTemplate ldapTemplate;
     private LdapContextSource ldapContextSource;
+    private PagedResultsDirContextProcessor processor;
 
     public TestAmbariLdapDataPopulator(Configuration configuration, Users users) {
       super(configuration, users);
@@ -85,10 +90,19 @@ public class AmbariLdapDataPopulatorTest {
       return ldapTemplate;
     }
 
+    @Override
+    protected PagedResultsDirContextProcessor createPagingProcessor() {
+      return processor;
+    }
+
     public void setLdapContextSource(LdapContextSource ldapContextSource) {
       this.ldapContextSource = ldapContextSource;
     }
 
+    public void setProcessor(PagedResultsDirContextProcessor processor) {
+      this.processor = processor;
+    }
+
     public void setLdapTemplate(LdapTemplate ldapTemplate) {
       this.ldapTemplate = ldapTemplate;
     }
@@ -1469,6 +1483,9 @@ public class AmbariLdapDataPopulatorTest {
     LdapTemplate ldapTemplate = createNiceMock(LdapTemplate.class);
     LdapServerProperties ldapServerProperties = createNiceMock(LdapServerProperties.class);
     Capture<ContextMapper> contextMapperCapture = new Capture<ContextMapper>();
+    Capture<SearchControls> searchControlsCapture = new Capture<SearchControls>();
+    PagedResultsDirContextProcessor processor = createNiceMock(PagedResultsDirContextProcessor.class);
+    PagedResultsCookie cookie = createNiceMock(PagedResultsCookie.class);
     LdapUserDto dto = new LdapUserDto();
 
     List<LdapUserDto> list = new LinkedList<LdapUserDto>();
@@ -1478,22 +1495,25 @@ public class AmbariLdapDataPopulatorTest {
     expect(ldapServerProperties.getUserObjectClass()).andReturn("objectClass").anyTimes();
     expect(ldapServerProperties.getDnAttribute()).andReturn("dn").anyTimes();
     expect(ldapServerProperties.getBaseDN()).andReturn("baseDN").anyTimes();
+    expect(processor.getCookie()).andReturn(cookie).anyTimes();
+    expect(cookie.getCookie()).andReturn(null).anyTimes();
 
     expect(ldapTemplate.lookup(eq("uid=foo,dc=example,dc=com"), capture(contextMapperCapture))).andReturn(dto);
 
     expect(ldapTemplate.lookup(eq("foo"), capture(contextMapperCapture))).andReturn(null);
-    expect(ldapTemplate.search(eq("baseDN"), eq("(&(objectClass=objectClass)(|(dn=foo)(uid=foo)))"),
capture(contextMapperCapture))).andReturn(list);
+    expect(ldapTemplate.search(eq("baseDN"), eq("(&(objectClass=objectClass)(|(dn=foo)(uid=foo)))"),
anyObject(SearchControls.class), capture(contextMapperCapture), eq(processor))).andReturn(list);
 
-    replay(ldapTemplate, ldapServerProperties, users, configuration);
+    replay(ldapTemplate, ldapServerProperties, users, configuration, processor, cookie);
 
     AmbariLdapDataPopulatorTestInstance populator = new AmbariLdapDataPopulatorTestInstance(configuration,
users);
 
     populator.setLdapTemplate(ldapTemplate);
+    populator.setProcessor(processor);
 
     assertEquals(dto, populator.getLdapUserByMemberAttr("uid=foo,dc=example,dc=com"));
     assertEquals(dto, populator.getLdapUserByMemberAttr("foo"));
 
-    verify(ldapTemplate, ldapServerProperties, users, configuration);
+    verify(ldapTemplate, ldapServerProperties, users, configuration, processor, cookie);
   }
 
   @Test


Mime
View raw message