usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From snoopd...@apache.org
Subject [23/23] incubator-usergrid git commit: Merge branch 'master' into two-dot-o
Date Wed, 22 Jul 2015 12:09:13 GMT
Merge branch 'master' into two-dot-o

Conflicts:
	CHANGELOG
	stack/rest/pom.xml
	stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
	stack/tools/src/main/resources/log4j.properties


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/764a7c85
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/764a7c85
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/764a7c85

Branch: refs/heads/two-dot-o
Commit: 764a7c85ebf4eeaa720209f46fe02fb96a47c760
Parents: 3b289aa e1b352e
Author: Dave Johnson <snoopdave@apache.org>
Authored: Wed Jul 22 08:08:36 2015 -0400
Committer: Dave Johnson <snoopdave@apache.org>
Committed: Wed Jul 22 08:08:36 2015 -0400

----------------------------------------------------------------------
 CHANGELOG                                       |   3 -
 LICENSE                                         |  49 +--
 NOTICE                                          |   3 +-
 .../css/arsmarquette/ARSMaquettePro-Light.otf   | Bin 184600 -> 0 bytes
 .../css/arsmarquette/ARSMaquettePro-Medium.otf  | Bin 188020 -> 0 bytes
 .../css/arsmarquette/ARSMaquettePro-Regular.otf | Bin 188096 -> 0 bytes
 portal/css/main.css                             |  15 -
 portal/css/main.min.css                         |   2 +-
 release/release-candidate.sh                    |  10 -
 stack/LICENSE-2.0.txt                           | 202 ------------
 .../applications/assets/AssetResourceIT.java    |   2 +-
 .../src/test/resources/cat-larger-than-6mb.jpg  | Bin 9799257 -> 0 bytes
 .../src/test/resources/ship-larger-than-6mb.gif | Bin 0 -> 7407487 bytes
 .../org/apache/usergrid/tools/ExportAdmins.java | 323 ++++++++++---------
 .../org/apache/usergrid/tools/ImportAdmins.java | 298 ++++++++---------
 .../org/apache/usergrid/tools/UserManager.java  |  22 ++
 stack/tools/src/main/resources/log4j.properties |   6 +-
 17 files changed, 378 insertions(+), 557 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/764a7c85/CHANGELOG
----------------------------------------------------------------------
diff --cc CHANGELOG
index 70ea277,5e692fc..47a080a
--- a/CHANGELOG
+++ b/CHANGELOG
@@@ -1,6 -1,15 +1,3 @@@
 -Usergrid 1.0.2
 ---------------------------------------------------------------------------------
 -
 -Usergrid 1.0.2
 ---------------------------------------------------------------------------------
 -
 -Usergrid 1.0.2
 ---------------------------------------------------------------------------------
 -
--Usergrid 1.0.2
----------------------------------------------------------------------------------
--
  
  Usergrid 1.0.2
  --------------------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/764a7c85/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
----------------------------------------------------------------------
diff --cc stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
index 0fb5149,b048327..b318065
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
@@@ -334,15 -313,13 +334,15 @@@ public class AssetResourceIT extends Ab
  
          try {
  
 -            UserRepo.INSTANCE.load( resource(), access_token );
 +            //UserRepo.INSTANCE.load( resource(), access_token );
  
-             byte[] data = IOUtils.toByteArray( this.getClass().getResourceAsStream( "/cat-larger-than-6mb.jpg"
) );
+             byte[] data = IOUtils.toByteArray( this.getClass().getResourceAsStream( "/ship-larger-than-6mb.gif"
) );
              FormDataMultiPart form = new FormDataMultiPart().field( "file", data, MediaType.MULTIPART_FORM_DATA_TYPE
);
  
 +            String orgAppPath = clientSetup.getOrganizationName() + "/" + clientSetup.getAppName();
 +
              // send data
 -            JsonNode node = resource().path( "/test-organization/test-app/bars" ).queryParam(
"access_token", access_token )
 +            JsonNode node = resource().path( orgAppPath + "/bars" ).queryParam( "access_token",
access_token )
                      .accept( MediaType.APPLICATION_JSON ).type( MediaType.MULTIPART_FORM_DATA
)
                      .post( JsonNode.class, form );
              //logNode( node );

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/764a7c85/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
----------------------------------------------------------------------
diff --cc stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
index 75064a2,2c14da1..e175d01
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
@@@ -14,40 -14,51 +14,47 @@@
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
- 
  package org.apache.usergrid.tools;
  
 -
  import com.google.common.collect.BiMap;
+ import com.google.common.collect.HashBiMap;
  import org.apache.commons.cli.CommandLine;
  import org.apache.commons.cli.Option;
  import org.apache.commons.cli.OptionBuilder;
  import org.apache.commons.cli.Options;
 +import org.apache.usergrid.corepersistence.util.CpNamingUtils;
- import org.apache.usergrid.persistence.ConnectionRef;
+ import org.apache.usergrid.management.UserInfo;
  import org.apache.usergrid.persistence.Entity;
  import org.apache.usergrid.persistence.EntityManager;
 -import org.apache.usergrid.persistence.Query;
  import org.apache.usergrid.persistence.Results;
--import org.apache.usergrid.persistence.Results.Level;
--import org.apache.usergrid.persistence.cassandra.CassandraService;
 +import org.apache.usergrid.persistence.index.query.Query;
  import org.apache.usergrid.utils.StringUtils;
  import org.codehaus.jackson.JsonGenerator;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  
- import java.util.List;
- import java.util.Map;
- import java.util.Set;
- import java.util.UUID;
- import java.util.concurrent.BlockingQueue;
- import java.util.concurrent.LinkedBlockingQueue;
- import java.util.concurrent.TimeUnit;
+ import java.util.*;
+ import java.util.concurrent.*;
+ import java.util.concurrent.atomic.AtomicInteger;
  
 -import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
 -
  
  /**
-  * Export Admin Users and metadata including organizations.
+  * Export Admin Users and metadata including organizations and passwords.
+  *
+  * Usage Example:
 - * 
++ *
+  * java -Xmx8000m -Dlog4j.configuration=file:/home/me/log4j.properties -classpath . \
+  *      -jar usergrid-tools-1.0.2.jar ImportAdmins -writeThreads 100 -auditThreads 100 \
+  *      -host casshost -inputDir=/home/me/export-data
 - * 
++ *
+  * If you want to provide any property overrides, put properties file named usergrid-custom-tools.properties
+  * in the same directory where you run the above command. For example, you might want to
set the Cassandra
+  * client threads and export from a specific set of keyspaces:
 - * 
 + *
-  * java -jar usergrid-tools.jar ExportAdmins
+  *    cassandra.connections=110
+  *    cassandra.system.keyspace=My_Usergrid
+  *    cassandra.application.keyspace=My_Usergrid_Applications
+  *    cassandra.lock.keyspace=My_Usergrid_Locks
   */
  public class ExportAdmins extends ExportingToolBase {
  
@@@ -55,9 -66,12 +62,12 @@@
      public static final String ADMIN_USERS_PREFIX = "admin-users";
      public static final String ADMIN_USER_METADATA_PREFIX = "admin-user-metadata";
      private static final String READ_THREAD_COUNT = "readThreads";
+     private Map<String, List<Org>> orgMap = new HashMap<String, List<Org>>(80000);
      private int readThreadCount;
 -    
 +
+     AtomicInteger count = new AtomicInteger( 0 );
 -   
 -    
++
 +
      /**
       * Represents an AdminUser that has been read and is ready for export.
       */
@@@ -69,8 -81,21 +77,21 @@@
          BiMap<UUID, String>              orgNamesByUuid;
      }
  
 -    
 +
      /**
+      * Represents an organization associated with a user.
+      */
+     private class Org {
+         UUID orgId;
+         String orgName;
+         public Org( UUID orgId, String orgName ) {
+             this.orgId = orgId;
+             this.orgName = orgName;
+         }
+     }
+ 
+ 
+     /**
       * Export admin users using multiple threads.
       * <p/>
       * How it works:
@@@ -100,6 -125,8 +121,8 @@@
              readThreadCount = 20;
          }
  
+         buildOrgMap();
 -                
++
          // start write queue worker
  
          BlockingQueue<AdminUserWriteTask> writeQueue = new LinkedBlockingQueue<AdminUserWriteTask>();
@@@ -124,8 -151,8 +147,8 @@@
  
          Query query = new Query();
          query.setLimit( MAX_ENTITY_FETCH );
--        query.setResultsLevel( Level.IDS );
--        EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID
);
++        query.setResultsLevel( Query.Level.IDS );
++        EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID
);
          Results ids = em.searchCollection( em.getApplicationRef(), "users", query );
  
          while (ids.size() > 0) {
@@@ -140,15 -167,16 +163,16 @@@
              ids = em.searchCollection( em.getApplicationRef(), "users", query );
          }
  
-         adminUserWriter.setDone( true );
-         for (AdminUserReader aur : readers) {
-             aur.setDone( true );
-         }
- 
          logger.debug( "Waiting for write thread to complete" );
-         writeThread.join();
 -        
++
+         boolean done = false;
+         while ( !done ) {
+             writeThread.join( 10000, 0 );
+             done = !writeThread.isAlive();
+             logger.info( "Wrote {} users", count.get() );
+         }
      }
 -    
 +
  
      @Override
      @SuppressWarnings("static-access")
@@@ -164,9 -192,77 +188,77 @@@
      }
  
  
-     public class AdminUserReader implements Runnable {
+     /**
+      * Shouldn't have to do this but getOrganizationsForAdminUser() is not 100% reliable
in some Usergrid installations.
+      */
+     private void buildOrgMap() throws Exception {
+ 
+         logger.info( "Building org map" );
+ 
+         ExecutorService execService = Executors.newFixedThreadPool( this.readThreadCount
);
+ 
 -        EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
++        EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID
);
+         String queryString = "select *";
+         Query query = Query.fromQL( queryString );
+         query.withLimit( 1000 );
+         Results organizations = null;
+         int count = 0;
+         do {
+             organizations = em.searchCollection( em.getApplicationRef(), "groups", query
);
+             for ( Entity organization : organizations.getEntities() ) {
+                 execService.submit( new OrgMapWorker( organization ) );
+             }
+             count++;
+             if ( count % 1000 == 0 ) {
+                 logger.info("Processed {} orgs for org map", count);
+             }
+             query.setCursor( organizations.getCursor() );
+         }
+         while ( organizations != null && organizations.hasCursor() );
+ 
+         execService.shutdown();
+         while ( !execService.awaitTermination( 10, TimeUnit.SECONDS ) ) {
+             logger.info("Processed {} orgs for map", orgMap.size() );
+         }
+     }
+ 
  
-         private boolean done = true;
+     public class OrgMapWorker implements Runnable {
+         private final Entity orgEntity;
 -        
++
+         public OrgMapWorker( Entity orgEntity ) {
+             this.orgEntity = orgEntity;
+         }
 -        
++
+         @Override
+         public void run() {
+             try {
+                 final String orgName = orgEntity.getProperty( "path" ).toString();
+                 final UUID orgId = orgEntity.getUuid();
+                 for (UserInfo user : managementService.getAdminUsersForOrganization( orgEntity.getUuid()
)) {
+                     try {
+                         Entity admin = managementService.getAdminUserEntityByUuid( user.getUuid()
);
+                         List<Org> orgs = orgMap.get( admin.getProperty( "username"
) );
+                         if (orgs == null) {
+                             orgs = new ArrayList<Org>();
+                             orgMap.put( admin.getProperty( "username" ).toString().toLowerCase(),
orgs );
+                         }
+                         orgs.add( new Org( orgId, orgName ) );
+ 
+                         //logger.debug("Added org {} for user {}", orgName, admin.getProperty(
"username" ));
+ 
+                     } catch (Exception e) {
+                         logger.warn( "Cannot get orgs for userId {}", user.getUuid() );
+                     }
+                 }
+             } catch ( Exception e ) {
+                 logger.error("Error getting users for org {}:{}", orgEntity.getName(), orgEntity.getUuid());
+             }
+         }
+     }
+ 
+ 
+     public class AdminUserReader implements Runnable {
  
          private final BlockingQueue<UUID> readQueue;
          private final BlockingQueue<AdminUserWriteTask> writeQueue;
@@@ -189,7 -285,7 +281,7 @@@
  
          private void readAndQueueAdminUsers() throws Exception {
  
--            EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID
);
++            EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID
);
  
              while ( true ) {
  
@@@ -221,11 -313,15 +309,15 @@@
          }
  
  
-         private void addCollectionsToTask(AdminUserWriteTask task, Entity entity) throws
Exception {
+         private void addDictionariesToTask(AdminUserWriteTask task, Entity entity) throws
Exception {
 -            EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID
);
 -            
++            EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID
);
++
+             task.dictionariesByName = new HashMap<String, Map<Object, Object>>();
+ 
+             Set<String> dictionaries = em.getDictionaries( entity );
 -            
 +
-             EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID
);
-             Set<String> collections = em.getCollections( entity );
-             if ((collections == null) || collections.isEmpty()) {
+             if ( dictionaries.isEmpty() ) {
+                 logger.error("User {}:{} has no dictionaries", task.adminUser.getName(),
task.adminUser.getUuid() );
                  return;
              }
  
@@@ -251,50 -344,26 +340,26 @@@
              }
          }
  
+         private void addOrganizationsToTask(AdminUserWriteTask task) throws Exception {
  
-         private void addDictionariesToTask(AdminUserWriteTask task, Entity entity) throws
Exception {
-             EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID
);
- 
-             Set<String> dictionaries = em.getDictionaries( entity );
- 
-             task.dictionariesByName = new HashMap<String, Map<Object, Object>>();
- 
-             for (String dictionary : dictionaries) {
-                 Map<Object, Object> dict = em.getDictionaryAsMap( entity, dictionary
);
-                 task.dictionariesByName.put( dictionary, dict );
-             }
-         }
- 
- 
-         private void addConnectionsToTask(AdminUserWriteTask task, Entity entity) throws
Exception {
-             EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID
);
- 
-             task.connectionsByType = new HashMap<String, List<ConnectionRef>>();
- 
-             Set<String> connectionTypes = em.getConnectionTypes( entity );
-             for (String connectionType : connectionTypes) {
- 
-                 List<ConnectionRef> connRefs = task.connectionsByType.get( connectionType
);
-                 if ( connRefs == null ) {
-                     connRefs = new ArrayList<ConnectionRef>();
-                 }
+             task.orgNamesByUuid = managementService.getOrganizationsForAdminUser( task.adminUser.getUuid()
);
  
-                 Results results = em.getConnectedEntities( entity.getUuid(), connectionType,
null, Level.IDS );
-                 List<ConnectionRef> connections = results.getConnections();
+             List<Org> orgs = orgMap.get( task.adminUser.getProperty( "username" ).toString().toLowerCase()
);
 -            
 +
-                 for (ConnectionRef connectionRef : connections) {
-                     connRefs.add( connectionRef );
+             if ( orgs != null && task.orgNamesByUuid.size() < orgs.size() ) {
+                 BiMap<UUID, String> bimap = HashBiMap.create();
+                 for (Org org : orgs) {
+                     bimap.put( org.orgId, org.orgName );
                  }
+                 task.orgNamesByUuid = bimap;
              }
-         }
- 
- 
-         private void addOrganizationsToTask(AdminUserWriteTask task, Entity entity) throws
Exception {
-             task.orgNamesByUuid = managementService.getOrganizationsForAdminUser( entity.getUuid()
);
-         }
 -            
 +
-         public void setDone(boolean done) {
-             this.done = done;
+             if ( task.orgNamesByUuid.isEmpty() ) {
+                 logger.error("{}:{}:{} has no orgs", new Object[] {
 -                        task.adminUser.getProperty("username"), 
 -                        task.adminUser.getProperty("email"), 
++                        task.adminUser.getProperty("username"),
++                        task.adminUser.getProperty("email"),
+                         task.adminUser.getUuid() } );
+             }
          }
      }
  
@@@ -320,7 -387,7 +383,7 @@@
  
  
          private void writeEntities() throws Exception {
--            EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID
);
++            EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID
);
  
              // write one JSON file for management application users
              JsonGenerator usersFile =
@@@ -347,18 -412,20 +408,20 @@@
                      echo( task.adminUser );
  
                      // write metadata to metadata file
-                     saveCollections(   metadataFile, task );
-                     saveConnections(   metadataFile, task );
+                     metadataFile.writeFieldName( task.adminUser.getUuid().toString() );
+                     metadataFile.writeStartObject();
+ 
                      saveOrganizations( metadataFile, task );
-                     saveDictionaries(  metadataFile, task );
+                     saveDictionaries( metadataFile, task );
 -                    
 +
-                     logger.debug("Exported user {}", task.adminUser.getProperty( "email"
));
+                     metadataFile.writeEndObject();
 -                    
 +
-                     count++;
-                     if ( count % 1000 == 0 ) {
-                         logger.info("Exported {} admin users", count);
-                     }
+                     logger.debug( "Exported user {}:{}:{}", new Object[] {
+                         task.adminUser.getProperty("username"),
+                         task.adminUser.getProperty("email"),
+                         task.adminUser.getUuid() } );
  
+                     count.addAndGet( 1 );
  
                  } catch (InterruptedException e) {
                      throw new Exception("Interrupted", e);

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/764a7c85/stack/tools/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --cc stack/tools/src/main/resources/log4j.properties
index c288fec,6cf0a92..00834cf
--- a/stack/tools/src/main/resources/log4j.properties
+++ b/stack/tools/src/main/resources/log4j.properties
@@@ -26,25 -26,9 +26,25 @@@ log4j.appender.stdout=org.apache.log4j.
  log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
  log4j.appender.stdout.layout.ConversionPattern=%d %p (%t) [%c] - %m%n
  
- <<<<<<< HEAD
 +log4j.category.org.apache.usergrid.tools=TRACE
 +log4j.category.org.apache.usergrid=ERROR
 +
 +log4j.logger.org.apache.usergrid.persistence.cassandra.DB=WARN, stdout
 +log4j.logger.org.apache.usergrid.persistence.cassandra.BATCH=WARN, stdout
 +log4j.logger.org.apache.usergrid.persistence.cassandra.EntityManagerFactoryImpl=WARN, stdout
 +log4j.logger.org.apache.usergrid.persistence.cassandra.DaoUtils=WARN, stdout
 +log4j.logger.org.apache.usergrid.persistence.cassandra.EntityManagerImpl=WARN, stdout
 +log4j.logger.org.apache.usergrid.persistence.cassandra.ConnectionRefImpl=WARN, stdout
 +log4j.logger.me.prettyprint.cassandra.hector.TimingLogger=WARN, stdout
 +log4j.logger.org.apache.usergrid.rest.security.AllowAjaxFilter=WARN, stdout
 +log4j.logger.me.prettyprint.hector.api.beans.AbstractComposite=ERROR, stdout
 +#log4j.logger.org.apache.usergrid.locking.singlenode.SingleNodeLockManagerImpl=DEBUG, stdout
 +#log4j.logger.org.apache.usergrid.persistence.hector.CountingMutator=INFO, stdout
- =======
- log4j.logger.org.apache.usergrid.tools=DEBUG
++
 +log4j.logger.org.apache.usergrid.management.cassandra=DEBUB
+ log4j.logger.org.apache.usergrid.tools=INFO
  
+ log4j.logger.org.apache.usergrid.management.cassandra=WARN
  log4j.logger.org.apache.usergrid.persistence.cassandra.DB=WARN
  log4j.logger.org.apache.usergrid.persistence.cassandra.BATCH=WARN
  log4j.logger.org.apache.usergrid.persistence.cassandra.EntityManagerFactoryImpl=WARN


Mime
View raw message