usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From snoopd...@apache.org
Subject [4/4] git commit: Changes to 1) ensure that the CpEntityManager's ElasticSearch can be rebuilt entirely from data stored in Cassandra, 2) provide support Index Rebuild in two-dot-o and the beginnings of an Index Rebuild test.
Date Wed, 01 Oct 2014 14:48:17 GMT
Changes to 1) ensure that the CpEntityManager's ElasticSearch can be rebuilt entirely from data stored in Cassandra, 2) provide support Index Rebuild in two-dot-o and the beginnings of an Index Rebuild test.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1aa04a71
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1aa04a71
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1aa04a71

Branch: refs/heads/two-dot-o-rebuildable-index
Commit: 1aa04a71d73b30f647b6d2d2ea8ce068557da31a
Parents: 58fc540
Author: Dave Johnson <dmjohnson@apigee.com>
Authored: Wed Oct 1 10:47:52 2014 -0400
Committer: Dave Johnson <dmjohnson@apigee.com>
Committed: Wed Oct 1 10:47:52 2014 -0400

----------------------------------------------------------------------
 .../corepersistence/CpEntityManager.java        | 479 +++++++++++---
 .../corepersistence/CpEntityManagerFactory.java | 639 ++++++-------------
 .../corepersistence/CpRelationManager.java      | 356 ++++-------
 .../usergrid/corepersistence/CpSetup.java       |  55 +-
 .../HybridEntityManagerFactory.java             |  15 +-
 .../persistence/EntityManagerFactory.java       |   9 +-
 .../persistence/cassandra/CassandraService.java |   5 -
 .../cassandra/EntityManagerFactoryImpl.java     |  54 +-
 .../PerformanceEntityRebuildIndexTest.java      |  64 +-
 .../org/apache/usergrid/tools/IndexRebuild.java |  62 +-
 10 files changed, 835 insertions(+), 903 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1aa04a71/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 51e660b..1c54d5f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -16,9 +16,13 @@
 package org.apache.usergrid.corepersistence;
 
 
+import com.netflix.hystrix.exception.HystrixRuntimeException;
+import com.yammer.metrics.annotation.Metered;
+import static java.lang.String.CASE_INSENSITIVE_ORDER;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
+import static java.util.Arrays.asList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
@@ -32,11 +36,26 @@ import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.util.Assert;
-
+import me.prettyprint.hector.api.Keyspace;
+import me.prettyprint.hector.api.beans.ColumnSlice;
+import me.prettyprint.hector.api.beans.CounterRow;
+import me.prettyprint.hector.api.beans.CounterRows;
+import me.prettyprint.hector.api.beans.CounterSlice;
+import me.prettyprint.hector.api.beans.DynamicComposite;
+import me.prettyprint.hector.api.beans.HColumn;
+import me.prettyprint.hector.api.beans.HCounterColumn;
+import me.prettyprint.hector.api.factory.HFactory;
+import static me.prettyprint.hector.api.factory.HFactory.createCounterSliceQuery;
+import static me.prettyprint.hector.api.factory.HFactory.createMutator;
+import me.prettyprint.hector.api.mutation.Mutator;
+import me.prettyprint.hector.api.query.MultigetSliceCounterQuery;
+import me.prettyprint.hector.api.query.QueryResult;
+import me.prettyprint.hector.api.query.SliceCounterQuery;
+import static org.apache.commons.lang.StringUtils.capitalize;
+import static org.apache.commons.lang.StringUtils.isBlank;
+import static org.apache.usergrid.corepersistence.CpRelationManager.ALL_TYPES;
+import static org.apache.usergrid.corepersistence.CpRelationManager.EDGE_COLL_SUFFIX;
+import static org.apache.usergrid.corepersistence.CpRelationManager.EDGE_CONN_SUFFIX;
 import org.apache.usergrid.persistence.AggregateCounter;
 import org.apache.usergrid.persistence.AggregateCounterSet;
 import org.apache.usergrid.persistence.CollectionRef;
@@ -50,14 +69,43 @@ import org.apache.usergrid.persistence.EntityRef;
 import org.apache.usergrid.persistence.IndexBucketLocator;
 import org.apache.usergrid.persistence.RelationManager;
 import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.Schema;
+import static org.apache.usergrid.persistence.Schema.COLLECTION_ROLES;
+import static org.apache.usergrid.persistence.Schema.COLLECTION_USERS;
+import static org.apache.usergrid.persistence.Schema.DICTIONARY_PERMISSIONS;
+import static org.apache.usergrid.persistence.Schema.DICTIONARY_ROLENAMES;
+import static org.apache.usergrid.persistence.Schema.DICTIONARY_ROLETIMES;
+import static org.apache.usergrid.persistence.Schema.DICTIONARY_SETS;
+import static org.apache.usergrid.persistence.Schema.PROPERTY_CREATED;
+import static org.apache.usergrid.persistence.Schema.PROPERTY_INACTIVITY;
+import static org.apache.usergrid.persistence.Schema.PROPERTY_MODIFIED;
+import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
+import static org.apache.usergrid.persistence.Schema.PROPERTY_TIMESTAMP;
+import static org.apache.usergrid.persistence.Schema.PROPERTY_TYPE;
+import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID;
+import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
+import static org.apache.usergrid.persistence.Schema.TYPE_ENTITY;
+import static org.apache.usergrid.persistence.Schema.getDefaultSchema;
 import org.apache.usergrid.persistence.SimpleEntityRef;
+import static org.apache.usergrid.persistence.SimpleEntityRef.getUuid;
+import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
+import org.apache.usergrid.persistence.SimpleRoleRef;
 import org.apache.usergrid.persistence.TypedEntity;
 import org.apache.usergrid.persistence.cassandra.ApplicationCF;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.APPLICATION_AGGREGATE_COUNTERS;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_COMPOSITE_DICTIONARIES;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_COUNTERS;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_DICTIONARIES;
 import org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils;
 import org.apache.usergrid.persistence.cassandra.CassandraService;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.ALL_COUNT;
 import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
 import org.apache.usergrid.persistence.cassandra.CounterUtils;
 import org.apache.usergrid.persistence.cassandra.GeoIndexManager;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
+import static org.apache.usergrid.persistence.cassandra.Serializers.le;
+import static org.apache.usergrid.persistence.cassandra.Serializers.se;
+import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
 import org.apache.usergrid.persistence.cassandra.util.TraceParticipant;
 import org.apache.usergrid.persistence.collection.CollectionScope;
 import org.apache.usergrid.persistence.collection.EntityCollectionManager;
@@ -73,6 +121,11 @@ import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsE
 import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
 import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException;
 import org.apache.usergrid.persistence.exceptions.UnexpectedEntityTypeException;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
 import org.apache.usergrid.persistence.index.EntityIndex;
 import org.apache.usergrid.persistence.index.IndexScope;
 import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
@@ -80,76 +133,27 @@ import org.apache.usergrid.persistence.index.query.CounterResolution;
 import org.apache.usergrid.persistence.index.query.Identifier;
 import org.apache.usergrid.persistence.index.query.Query;
 import org.apache.usergrid.persistence.index.query.Query.Level;
+import static org.apache.usergrid.persistence.index.query.Query.Level.REFS;
 import org.apache.usergrid.persistence.model.entity.Id;
 import org.apache.usergrid.persistence.model.entity.SimpleId;
 import org.apache.usergrid.persistence.model.field.Field;
 import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.persistence.schema.CollectionInfo;
 import org.apache.usergrid.utils.ClassUtils;
-import org.apache.usergrid.utils.CompositeUtils;
-import org.apache.usergrid.utils.StringUtils;
-import org.apache.usergrid.utils.UUIDUtils;
-
-import com.netflix.hystrix.exception.HystrixRuntimeException;
-import com.yammer.metrics.annotation.Metered;
-
-import me.prettyprint.hector.api.Keyspace;
-import me.prettyprint.hector.api.beans.ColumnSlice;
-import me.prettyprint.hector.api.beans.CounterRow;
-import me.prettyprint.hector.api.beans.CounterRows;
-import me.prettyprint.hector.api.beans.CounterSlice;
-import me.prettyprint.hector.api.beans.DynamicComposite;
-import me.prettyprint.hector.api.beans.HColumn;
-import me.prettyprint.hector.api.beans.HCounterColumn;
-import me.prettyprint.hector.api.factory.HFactory;
-import me.prettyprint.hector.api.mutation.Mutator;
-import me.prettyprint.hector.api.query.MultigetSliceCounterQuery;
-import me.prettyprint.hector.api.query.QueryResult;
-import me.prettyprint.hector.api.query.SliceCounterQuery;
-import rx.Observable;
-
-import static java.lang.String.CASE_INSENSITIVE_ORDER;
-import static java.util.Arrays.asList;
-
-import static me.prettyprint.hector.api.factory.HFactory.createCounterSliceQuery;
-import static me.prettyprint.hector.api.factory.HFactory.createMutator;
-import static org.apache.commons.lang.StringUtils.capitalize;
-import static org.apache.commons.lang.StringUtils.isBlank;
-import static org.apache.usergrid.corepersistence.CpRelationManager.ALL_TYPES;
-import static org.apache.usergrid.persistence.Schema.COLLECTION_ROLES;
-import static org.apache.usergrid.persistence.Schema.COLLECTION_USERS;
-import static org.apache.usergrid.persistence.Schema.DICTIONARY_PERMISSIONS;
-import static org.apache.usergrid.persistence.Schema.DICTIONARY_ROLENAMES;
-import static org.apache.usergrid.persistence.Schema.DICTIONARY_ROLETIMES;
-import static org.apache.usergrid.persistence.Schema.DICTIONARY_SETS;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_CREATED;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_INACTIVITY;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_MODIFIED;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_TIMESTAMP;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_TYPE;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID;
-import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
-import static org.apache.usergrid.persistence.Schema.TYPE_ENTITY;
-import org.apache.usergrid.persistence.Schema;
-import static org.apache.usergrid.persistence.SimpleEntityRef.getUuid;
-import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
-import org.apache.usergrid.persistence.SimpleRoleRef;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.APPLICATION_AGGREGATE_COUNTERS;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_COMPOSITE_DICTIONARIES;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_COUNTERS;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_DICTIONARIES;
-import static org.apache.usergrid.persistence.cassandra.CassandraService.ALL_COUNT;
-import static org.apache.usergrid.persistence.cassandra.Serializers.be;
-import static org.apache.usergrid.persistence.cassandra.Serializers.le;
-import static org.apache.usergrid.persistence.cassandra.Serializers.se;
-import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
-import static org.apache.usergrid.persistence.index.query.Query.Level.REFS;
 import static org.apache.usergrid.utils.ClassUtils.cast;
+import org.apache.usergrid.utils.CompositeUtils;
 import static org.apache.usergrid.utils.ConversionUtils.bytebuffer;
 import static org.apache.usergrid.utils.ConversionUtils.getLong;
 import static org.apache.usergrid.utils.ConversionUtils.object;
 import static org.apache.usergrid.utils.ConversionUtils.string;
 import static org.apache.usergrid.utils.InflectionUtils.singularize;
+import org.apache.usergrid.utils.StringUtils;
+import org.apache.usergrid.utils.UUIDUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.Assert;
+import rx.Observable;
+import rx.functions.Action1;
 
 
 
@@ -172,7 +176,7 @@ public class CpEntityManager implements EntityManager {
 
     private CpManagerCache managerCache;
 
-    private ApplicationScope appScope;
+    private ApplicationScope applicationScope;
 
     private CassandraService cass;
 
@@ -191,7 +195,7 @@ public class CpEntityManager implements EntityManager {
         this.managerCache = this.emf.getManagerCache();
         this.applicationId = applicationId;
 
-        appScope = this.emf.getApplicationScope( applicationId );
+        applicationScope = this.emf.getApplicationScope( applicationId );
 
         this.cass = this.emf.cass;
         this.counterUtils = this.emf.counterUtils;
@@ -200,7 +204,7 @@ public class CpEntityManager implements EntityManager {
         this.skipAggregateCounters = false;
 
 
-        appScope = this.emf.getApplicationScope( applicationId );
+        applicationScope = this.emf.getApplicationScope( applicationId );
     }
 
 
@@ -226,6 +230,66 @@ public class CpEntityManager implements EntityManager {
         return csn.toLowerCase();
     }
 
+    static boolean isCollectionEdgeType( String type )  {
+        return type.endsWith( EDGE_COLL_SUFFIX );
+    }
+    
+    static boolean isConnectionEdgeType( String type )  {
+        return type.endsWith( EDGE_CONN_SUFFIX );
+    }
+    
+    static public String getConnectionType( String edgeType ) {
+        String[] parts = edgeType.split("\\|");
+        return parts[0];
+    }
+
+    static public String getCollectionEntityType( String edgeType ) {
+        String[] parts = edgeType.split("\\|");
+        return parts[1];
+    }
+
+    static public String getCollectionName( String edgeType ) {
+        String[] parts = edgeType.split("\\|");
+        return parts[0];
+    }
+
+    static public String getConnectedEntityType( String edgeType ) {
+        String[] parts = edgeType.split("\\|");
+        return parts[1];
+    }
+
+    static String getEdgeTypeFromConnectionType( String connectionType, String targetEntityType ) {
+
+        if ( connectionType != null && targetEntityType != null ) {
+            String csn = connectionType + "|" + targetEntityType + "|" + EDGE_CONN_SUFFIX;
+            return csn;
+        }
+
+        if ( connectionType != null ) {
+            // no suffix, this must be a search
+            String csn = connectionType;
+            return csn;
+        } 
+
+        return null;
+    }
+
+    static String getEdgeTypeFromCollectionName( String collectionName, String targetEntityType ) {
+
+        if ( collectionName != null && targetEntityType != null ) {
+            String csn = collectionName + "|" + targetEntityType + "|" + EDGE_COLL_SUFFIX;
+            return csn;
+        }
+
+        if ( collectionName != null ) {
+            // no suffix, this must be a search
+            String csn = collectionName;
+            return csn;
+        } 
+
+        return null;
+    }
+
 
     @Override
     public Entity create( String entityType, Map<String, Object> properties ) throws Exception {
@@ -318,7 +382,7 @@ public class CpEntityManager implements EntityManager {
         String collectionName = getCollectionScopeNameFromEntityType( entityRef.getType() );
 
         CollectionScope collectionScope = new CollectionScopeImpl( 
-                appScope.getApplication(), appScope.getApplication(), collectionName );
+                applicationScope.getApplication(), applicationScope.getApplication(), collectionName );
 
         EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
 
@@ -405,7 +469,7 @@ public class CpEntityManager implements EntityManager {
         String collectionName = getCollectionScopeNameFromEntityType( type );
 
         CollectionScope collectionScope = new CollectionScopeImpl( 
-                appScope.getApplication(), appScope.getApplication(), collectionName );
+                applicationScope.getApplication(), applicationScope.getApplication(), collectionName );
 
         EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
 
@@ -471,8 +535,8 @@ public class CpEntityManager implements EntityManager {
 
         // first, update entity index in its own collection scope
         CollectionScope collectionScope = new CollectionScopeImpl( 
-            appScope.getApplication(), 
-            appScope.getApplication(),
+            applicationScope.getApplication(), 
+            applicationScope.getApplication(),
             getCollectionScopeNameFromEntityType( entity.getType() ) );
         EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
 
@@ -534,8 +598,8 @@ public class CpEntityManager implements EntityManager {
     private Observable deleteAsync( EntityRef entityRef ) throws Exception {
 
         CollectionScope collectionScope = new CollectionScopeImpl( 
-            appScope.getApplication(), 
-            appScope.getApplication(),
+            applicationScope.getApplication(), 
+            applicationScope.getApplication(),
             getCollectionScopeNameFromEntityType( entityRef.getType() ) );
 
         EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
@@ -568,7 +632,7 @@ public class CpEntityManager implements EntityManager {
                     for ( String coll : collectionNames ) {
 
                         IndexScope indexScope = new IndexScopeImpl( 
-                                appScope.getApplication(), 
+                                applicationScope.getApplication(), 
                                 new SimpleId( uuid, ownerType ), 
                                 CpEntityManager.getCollectionScopeNameFromCollectionName(coll) );
 
@@ -581,15 +645,15 @@ public class CpEntityManager implements EntityManager {
 
             // deindex from default index scope
             IndexScope defaultIndexScope = new IndexScopeImpl( 
-                    appScope.getApplication(), 
-                    appScope.getApplication(),
+                    applicationScope.getApplication(), 
+                    applicationScope.getApplication(),
                     getCollectionScopeNameFromEntityType( entityRef.getType() ) );
             EntityIndex entityIndex = managerCache.getEntityIndex( defaultIndexScope );
             entityIndex.deindex( entity );
 
             IndexScope allTypesIndexScope = new IndexScopeImpl( 
-                appScope.getApplication(), 
-                appScope.getApplication(), 
+                applicationScope.getApplication(), 
+                applicationScope.getApplication(), 
                 ALL_TYPES);
             EntityIndex aei = managerCache.getEntityIndex( allTypesIndexScope );
             aei.deindex( entity );
@@ -969,13 +1033,13 @@ public class CpEntityManager implements EntityManager {
         String collectionName = getCollectionScopeNameFromEntityType( entityRef.getType() );
 
         CollectionScope collectionScope = new CollectionScopeImpl( 
-                appScope.getApplication(), 
-                appScope.getApplication(), 
+                applicationScope.getApplication(), 
+                applicationScope.getApplication(), 
                 collectionName );
 
         IndexScope defaultIndexScope = new IndexScopeImpl( 
-                appScope.getApplication(), 
-                appScope.getApplication(), 
+                applicationScope.getApplication(), 
+                applicationScope.getApplication(), 
                 getCollectionScopeNameFromEntityType( entityRef.getType()) );
 
         EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
@@ -1238,7 +1302,8 @@ public class CpEntityManager implements EntityManager {
 
         ColumnSlice<ByteBuffer, ByteBuffer> results =
                 cass.getColumns( cass.getApplicationKeyspace( applicationId ), dictionaryCf,
-                        CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ), columnNames, be, be );
+                        CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ), 
+                        columnNames, be, be );
         if ( results != null ) {
             values = new HashMap<String, Object>();
             for ( HColumn<ByteBuffer, ByteBuffer> result : results.getColumns() ) {
@@ -1618,11 +1683,14 @@ public class CpEntityManager implements EntityManager {
 
         Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be );
         CassandraPersistenceUtils.addInsertToMutator( batch, ENTITY_DICTIONARIES, 
-                CassandraPersistenceUtils.key( ownerId, Schema.DICTIONARY_ROLENAMES ), roleName, roleTitle, timestamp );
+                CassandraPersistenceUtils.key( ownerId, Schema.DICTIONARY_ROLENAMES ), 
+                roleName, roleTitle, timestamp );
         CassandraPersistenceUtils.addInsertToMutator( batch, ENTITY_DICTIONARIES, 
-                CassandraPersistenceUtils.key( ownerId, Schema.DICTIONARY_ROLETIMES ), roleName, inactivity, timestamp );
+                CassandraPersistenceUtils.key( ownerId, Schema.DICTIONARY_ROLETIMES ), 
+                roleName, inactivity, timestamp );
         CassandraPersistenceUtils.addInsertToMutator( batch, ENTITY_DICTIONARIES, 
-                CassandraPersistenceUtils.key( ownerId, DICTIONARY_SETS ), Schema.DICTIONARY_ROLENAMES, null, timestamp );
+                CassandraPersistenceUtils.key( ownerId, DICTIONARY_SETS ), 
+                Schema.DICTIONARY_ROLENAMES, null, timestamp );
 
         CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
 
@@ -1662,13 +1730,15 @@ public class CpEntityManager implements EntityManager {
 
 
     private Object getRolePermissionsKey( String roleName ) {
-        return CassandraPersistenceUtils.key( SimpleRoleRef.getIdForRoleName( roleName ), DICTIONARY_PERMISSIONS );
+        return CassandraPersistenceUtils.key( SimpleRoleRef.getIdForRoleName( roleName ), 
+                DICTIONARY_PERMISSIONS );
     }
 
 
     private Object getRolePermissionsKey( UUID groupId, String roleName ) {
         try {
-            return CassandraPersistenceUtils.key( getGroupRoleRef( groupId, roleName ).getUuid(), DICTIONARY_PERMISSIONS );
+            return CassandraPersistenceUtils.key( getGroupRoleRef( groupId, roleName ).getUuid(), 
+                    DICTIONARY_PERMISSIONS );
         } catch ( Exception e ) {
             logger.error("Error creating role key for uuid {} and role {}", groupId, roleName );
             return null;
@@ -2485,8 +2555,8 @@ public class CpEntityManager implements EntityManager {
 
         // prepare to write and index Core Persistence Entity into default scope
         CollectionScope collectionScope = new CollectionScopeImpl( 
-                appScope.getApplication(), 
-                appScope.getApplication(),
+                applicationScope.getApplication(), 
+                applicationScope.getApplication(),
                 getCollectionScopeNameFromEntityType( eType ) );
         EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
 
@@ -2530,8 +2600,8 @@ public class CpEntityManager implements EntityManager {
 
         // Index CP entity into default collection scope
         IndexScope defaultIndexScope = new IndexScopeImpl( 
-            appScope.getApplication(), 
-            appScope.getApplication(), 
+            applicationScope.getApplication(), 
+            applicationScope.getApplication(), 
             CpEntityManager.getCollectionScopeNameFromEntityType( entity.getType() ) );
         EntityIndex ei = managerCache.getEntityIndex( defaultIndexScope );
         ei.index( cpEntity );
@@ -2644,16 +2714,18 @@ public class CpEntityManager implements EntityManager {
             if ( !removeFromDictionary ) {
                 // Set the new value
 
-                elementCoValue = CassandraPersistenceUtils.toStorableBinaryValue( elementCoValue, !entityHasDictionary );
+                elementCoValue = CassandraPersistenceUtils.toStorableBinaryValue( 
+                        elementCoValue, !entityHasDictionary );
 
-                CassandraPersistenceUtils.addInsertToMutator( 
-                        batch, dictionary_cf, CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ),
+                CassandraPersistenceUtils.addInsertToMutator( batch, dictionary_cf, 
+                        CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ),
                     entityHasDictionary 
                         ? elementValue : asList( elementValue ), elementCoValue, timestamp );
 
                 if ( !entityHasDictionary ) {
                     CassandraPersistenceUtils.addInsertToMutator( batch, ENTITY_DICTIONARIES, 
-                        CassandraPersistenceUtils.key( entity.getUuid(), DICTIONARY_SETS ), dictionaryName, null, timestamp );
+                        CassandraPersistenceUtils.key( entity.getUuid(), DICTIONARY_SETS ), 
+                        dictionaryName, null, timestamp );
                 }
             }
             else {
@@ -2749,14 +2821,15 @@ public class CpEntityManager implements EntityManager {
 
         // refresh this Entity Manager's application's index
         IndexScope indexScope = new IndexScopeImpl( 
-                appScope.getApplication(), appScope.getApplication(), "dummy" );
+                applicationScope.getApplication(), applicationScope.getApplication(), "dummy" );
 
         EntityIndex ei = managerCache.getEntityIndex( indexScope );
         ei.refresh();
     }
 
 
-    public static org.apache.usergrid.persistence.model.entity.Entity entityToCpEntity( Entity entity, UUID importId ) {
+    public static org.apache.usergrid.persistence.model.entity.Entity 
+        entityToCpEntity( Entity entity, UUID importId ) {
 
         UUID uuid = importId != null ? importId : entity.getUuid();
 
@@ -2777,6 +2850,216 @@ public class CpEntityManager implements EntityManager {
     public void flushManagerCaches() {
         managerCache.flush();
     }
+
+
+    /** 
+     * Completely reindex the application associated with this EntityManager.
+     */
+    public void reindex( EntityManagerFactory.ProgressObserver po ) throws Exception {
+        indexEntityConnectionsAndCollections( getApplication(), po );
+    }
+
+
+    /**
+     * Recursively index (or reindex) all of the collections and connections of a 
+     * specified entity, and all of the collected and connected entities as well.
+     */
+    private void indexEntityConnectionsAndCollections( 
+            final EntityRef entity, final EntityManagerFactory.ProgressObserver po ) {
+
+        final GraphManager gm = managerCache.getGraphManager(applicationScope);
+
+        final Id fromEntityId = new SimpleId( entity.getUuid(), entity.getType() );
+
+        logger.debug("Loading edges types from {}:{}\n   scope {}:{}", 
+                new Object[] { entity.getType(), entity.getUuid(), 
+                applicationScope.getApplication().getType(), 
+                applicationScope.getApplication().getUuid() } );
+
+        Observable<String> edgeTypes = gm.getEdgeTypesFromSource( 
+                new SimpleSearchEdgeType( fromEntityId, null , null ));
+
+        edgeTypes.forEach( new Action1<String>() {
+
+            @Override
+            public void call( final String edgeType ) {
+
+                logger.debug("Loading edges of edgeType {} from {}:{}\n   scope {}:{}", 
+                        new Object[] { edgeType, entity.getType(), entity.getUuid(),
+                        applicationScope.getApplication().getType(), 
+                        applicationScope.getApplication().getUuid() } );
+
+                Observable<Edge> edges = gm.loadEdgesFromSource( new SimpleSearchByEdgeType( 
+                        fromEntityId, edgeType, Long.MAX_VALUE, 
+                        SearchByEdgeType.Order.DESCENDING, null ));
+
+                edges.forEach( new Action1<Edge>() {
+
+                    @Override
+                    public void call( Edge edge ) {
+
+                        if ( isCollectionEdgeType( edge.getType() )) {
+
+                            String collName = getCollectionName(edgeType);
+                            String memberType = getCollectionEntityType(edgeType);
+
+                            CollectionScope collScope = new CollectionScopeImpl(
+                                applicationScope.getApplication(),
+                                applicationScope.getApplication(),
+                                CpEntityManager.getCollectionScopeNameFromCollectionName(collName));
+                            EntityCollectionManager collMgr = 
+                                managerCache.getEntityCollectionManager(collScope);
+
+                            org.apache.usergrid.persistence.model.entity.Entity collEntity = 
+                                collMgr.load( edge.getTargetNode()).toBlockingObservable().last();
+
+                            CollectionScope memberScope = new CollectionScopeImpl(
+                                applicationScope.getApplication(),
+                                applicationScope.getApplication(),
+                                CpEntityManager.getCollectionScopeNameFromEntityType(memberType));
+                            EntityCollectionManager memberMgr = 
+                                managerCache.getEntityCollectionManager(memberScope);
+
+                            org.apache.usergrid.persistence.model.entity.Entity memberEntity = 
+                                memberMgr.load( edge.getTargetNode()).toBlockingObservable().last();
+
+                            indexEntityIntoCollections( collEntity, memberEntity, collName, true );
+
+                            EntityRef ref = new SimpleEntityRef( 
+                                    memberEntity.getId().getType(), memberEntity.getId().getUuid());
+                            po.onProgress( entity, ref, edge.getType());
+
+                            // recursion
+                            indexEntityConnectionsAndCollections( new SimpleEntityRef(
+                                memberEntity.getId().getType(), memberEntity.getId().getUuid()),po);
+
+                        } else if ( isConnectionEdgeType( edge.getType() )) {
+
+                            String connType = getConnectionType(edgeType);
+                            String targetEntityType = getConnectedEntityType(edgeType);
+                            String sourceEntityType = entity.getType();
+
+                            CollectionScope sourceScope = new CollectionScopeImpl(
+                                applicationScope.getApplication(),
+                                applicationScope.getApplication(),
+                                CpEntityManager.getCollectionScopeNameFromEntityType(sourceEntityType));
+                            EntityCollectionManager sourceEcm = 
+                                managerCache.getEntityCollectionManager(sourceScope);
+
+                            org.apache.usergrid.persistence.model.entity.Entity sourceEntity = 
+                                sourceEcm.load( edge.getTargetNode()).toBlockingObservable().last();
+
+                            CollectionScope targetScope = new CollectionScopeImpl(
+                                applicationScope.getApplication(),
+                                applicationScope.getApplication(),
+                                CpEntityManager.getCollectionScopeNameFromEntityType(targetEntityType));
+                            EntityCollectionManager targetEcm = 
+                                managerCache.getEntityCollectionManager(targetScope);
+
+                            org.apache.usergrid.persistence.model.entity.Entity targetEntity = 
+                                targetEcm.load( edge.getTargetNode() ).toBlockingObservable().last();
+
+                            indexEntityIntoConnection( 
+                                    sourceEntity, targetEntity, targetEntityType, connType );
+
+                            EntityRef ref = new SimpleEntityRef( 
+                                targetEntity.getId().getType(), targetEntity.getId().getUuid());
+                            po.onProgress( entity, ref, edge.getType());
+
+                            // recursion
+                            indexEntityConnectionsAndCollections( new SimpleEntityRef(
+                                targetEntity.getId().getType(), targetEntity.getId().getUuid()),po);
+                        }
+                    }
+
+                }); // end foreach on edges
+
+            }
+
+        }); // end foreach on edgeTypes
+
+    }
+
+
+    private void indexEntityIntoCollections( 
+            org.apache.usergrid.persistence.model.entity.Entity collectionEntity, 
+            org.apache.usergrid.persistence.model.entity.Entity memberEntity, 
+            String collName, 
+            boolean connectBack ) {
+
+        logger.debug("Indexing into collections {} {}:{} member {}:{}", new Object[] { 
+            collName, collectionEntity.getId().getType(), collectionEntity.getId().getUuid(),
+            memberEntity.getId().getType(), memberEntity.getId().getUuid() });
+
+        indexEntityIntoCollection( collectionEntity, memberEntity, collName );
+
+        CollectionInfo collection = getDefaultSchema()
+                .getCollection( memberEntity.getId().getType(), collName);
+
+        if (connectBack && collection != null && collection.getLinkedCollection() != null) {
+            indexEntityIntoCollections( 
+                memberEntity, collectionEntity, collection.getLinkedCollection(), false );
+        }
+    }
+
+
+    void indexEntityIntoConnection(
+            org.apache.usergrid.persistence.model.entity.Entity sourceEntity,
+            org.apache.usergrid.persistence.model.entity.Entity targetEntity,
+            String targetEntityType,
+            String connType ) {
+
+        logger.debug("Indexing into connection {} source {}:{} target {}:{}", new Object[] { 
+            connType, sourceEntity.getId().getType(), sourceEntity.getId().getUuid(),
+            targetEntity.getId().getType(), targetEntity.getId().getUuid() });
+
+        // Index the new connection in app|source|type context
+        IndexScope indexScope = new IndexScopeImpl(
+                applicationScope.getApplication(),
+                sourceEntity.getId(),
+                CpEntityManager.getConnectionScopeName(targetEntityType, connType));
+        EntityIndex ei = managerCache.getEntityIndex(indexScope);
+        ei.index(targetEntity);
+        
+        // Index the new connection in app|scope|all-types context
+        IndexScope allTypesIndexScope = new IndexScopeImpl(
+                applicationScope.getApplication(),
+                sourceEntity.getId(),
+                ALL_TYPES);
+        EntityIndex aei = managerCache.getEntityIndex(allTypesIndexScope);
+        aei.index(targetEntity);
+    }
+
+
+    void indexEntityIntoCollection(
+            org.apache.usergrid.persistence.model.entity.Entity collectionEntity, 
+            org.apache.usergrid.persistence.model.entity.Entity memberEntity, 
+            String collName ) {
+
+        // index member into entity collection | type scope
+        IndexScope collectionIndexScope = new IndexScopeImpl(
+                applicationScope.getApplication(),
+                collectionEntity.getId(),
+                CpEntityManager.getCollectionScopeNameFromCollectionName(collName));
+        EntityIndex collectionIndex = managerCache.getEntityIndex(collectionIndexScope);
+        collectionIndex.index(memberEntity);
+        
+        // index member into entity | all-types scope
+        IndexScope entityAllTypesScope = new IndexScopeImpl(
+                applicationScope.getApplication(),
+                collectionEntity.getId(),
+                ALL_TYPES);
+        EntityIndex entityAllCollectionIndex = managerCache.getEntityIndex(entityAllTypesScope);
+        entityAllCollectionIndex.index(memberEntity);
+        
+        // index member into application | all-types scope
+        IndexScope appAllTypesScope = new IndexScopeImpl(
+                applicationScope.getApplication(),
+                applicationScope.getApplication(),
+                ALL_TYPES);
+        EntityIndex allCollectionIndex = managerCache.getEntityIndex(appAllTypesScope);
+        allCollectionIndex.index(memberEntity);
+    }
 }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1aa04a71/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 5ca9536..168c1bf 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -24,47 +24,40 @@ import static java.lang.String.CASE_INSENSITIVE_ORDER;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
-import java.util.Set;
 import java.util.TreeMap;
 import java.util.UUID;
 import org.apache.commons.lang.StringUtils;
+import org.apache.usergrid.persistence.AbstractEntity;
 import org.apache.usergrid.persistence.DynamicEntity;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityFactory;
 import org.apache.usergrid.persistence.EntityManager;
 import org.apache.usergrid.persistence.EntityManagerFactory;
 import org.apache.usergrid.persistence.Results;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_CREATED;
 import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID;
 import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
 import org.apache.usergrid.persistence.cassandra.CassandraService;
 import org.apache.usergrid.persistence.cassandra.CounterUtils;
 import org.apache.usergrid.persistence.cassandra.Setup;
 import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
 import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
 import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
 import org.apache.usergrid.persistence.entities.Application;
 import org.apache.usergrid.persistence.exceptions.ApplicationAlreadyExistsException;
-import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
+import org.apache.usergrid.persistence.graph.Edge;
 import org.apache.usergrid.persistence.graph.GraphManager;
 import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
-import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
 import org.apache.usergrid.persistence.index.EntityIndexFactory;
 import org.apache.usergrid.persistence.index.IndexScope;
-import org.apache.usergrid.persistence.index.query.CandidateResult;
 import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
-import org.apache.usergrid.persistence.index.query.CandidateResults;
 import org.apache.usergrid.persistence.index.query.Query;
-import org.apache.usergrid.persistence.model.entity.Entity;
 import org.apache.usergrid.persistence.model.entity.Id;
 import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.field.Field;
-import org.apache.usergrid.persistence.model.field.LongField;
-import org.apache.usergrid.persistence.model.field.StringField;
-import org.apache.usergrid.persistence.model.field.UUIDField;
+import org.apache.usergrid.utils.UUIDUtils;
 import org.apache.usergrid.persistence.model.util.UUIDGenerator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -93,7 +86,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
     // The System Application where we store app and org metadata
     public static final UUID SYSTEM_APP_ID = 
             UUID.fromString("b6768a08-b5d5-11e3-a495-10ddb1de66c3");
-    
+
     public static final  UUID MANAGEMENT_APPLICATION_ID = 
             UUID.fromString("b6768a08-b5d5-11e3-a495-11ddb1de66c8");
 
@@ -101,44 +94,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
             UUID.fromString("b6768a08-b5d5-11e3-a495-11ddb1de66c9");
 
 
-    @Deprecated // use system app for these in future
-    public static final String SYSTEM_APPS_TYPE = "zzzappszzz";
-
-    @Deprecated 
-    public static final String SYSTEM_ORGS_TYPE = "zzzorgszzz";
-    
-    @Deprecated 
-    public static final String SYSTEM_PROPS_TYPE = "zzzpropszzz"; 
-
-    @Deprecated // use system app for these in future
-    private static final Id systemAppId = 
-         new SimpleId( SYSTEM_APP_ID, SYSTEM_APPS_TYPE );
-    
-    @Deprecated 
-    public static final CollectionScope SYSTEM_APPS_SCOPE = 
-        new CollectionScopeImpl( systemAppId, systemAppId, SYSTEM_APPS_TYPE );
-
-    @Deprecated 
-    public static final IndexScope SYSTEM_APPS_INDEX_SCOPE = 
-        new IndexScopeImpl( systemAppId, systemAppId,  SYSTEM_APPS_TYPE);
-
-    @Deprecated 
-    public static final CollectionScope SYSTEM_ORGS_SCOPE = 
-        new CollectionScopeImpl( systemAppId, systemAppId,  SYSTEM_ORGS_TYPE);
-
-    @Deprecated
-    public static final IndexScope SYSTEM_ORGS_INDEX_SCOPE = 
-        new IndexScopeImpl( systemAppId, systemAppId, SYSTEM_ORGS_TYPE);
-
-    @Deprecated
-    public static final CollectionScope SYSTEM_PROPS_SCOPE = 
-        new CollectionScopeImpl( systemAppId, systemAppId, SYSTEM_PROPS_TYPE);
-
-    @Deprecated
-    public static final IndexScope SYSTEM_PROPS_INDEX_SCOPE = 
-        new IndexScopeImpl( systemAppId, systemAppId, SYSTEM_PROPS_TYPE);
-
-
     // cache of already instantiated entity managers
     private LoadingCache<UUID, EntityManager> entityManagers
         = CacheBuilder.newBuilder().maximumSize(100).build(new CacheLoader<UUID, EntityManager>() {
@@ -168,21 +123,28 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
             logger.warn("NOTE: Counters have been disabled by configuration...");
         }
 
-        // if system app does have apps, orgs and props then populate it
+    }
+    
+
+    private void init() {
+
+        EntityManager em = getEntityManager(SYSTEM_APP_ID);
+
         try {
-            EntityManager em = getEntityManager(SYSTEM_APP_ID);
-            Results orgs = em.searchCollection(em.getApplicationRef(), "organizations", null);
-            if (orgs.isEmpty()) {
-                populateSystemAppsFromEs();
-                populateSystemOrgsFromEs();
-                populateSystemPropsFromEs();
+            if ( em.getApplication() == null ) {
+                logger.info("Creating system application");
+                Map sysAppProps = new HashMap<String, Object>();
+                sysAppProps.put( PROPERTY_NAME, "systemapp");
+                em.create(SYSTEM_APP_ID, TYPE_APPLICATION, sysAppProps );
+                em.getApplication();
+                em.refreshIndex();
             }
 
         } catch (Exception ex) {
-            throw new RuntimeException("Fatal error migrating data", ex);
+            throw new RuntimeException("Fatal error creating system application", ex);
         }
     }
-    
+
 
     public CpManagerCache getManagerCache() {
 
@@ -267,7 +229,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
     public UUID initializeApplication( String organizationName, UUID applicationId, String name,
                                        Map<String, Object> properties ) throws Exception {
 
-        String appName = buildAppName( organizationName, name );
+        
+        EntityManager em = getEntityManager(SYSTEM_APP_ID);
+
+        final String appName = buildAppName( organizationName, name );
 
         // check for pre-existing application
         if ( lookupApplication( appName ) != null ) {
@@ -278,58 +243,38 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
 
         UUID orgUuid = lookupOrganization( organizationName );
         if ( orgUuid == null ) {
-          
-            // organization does not exist, create it.
-            Entity orgInfoEntity = new Entity(generateOrgId( UUIDGenerator.newTimeUUID() ));
 
-            orgUuid = orgInfoEntity.getId().getUuid();
-
-            long timestamp = System.currentTimeMillis();
-            orgInfoEntity.setField( new LongField( PROPERTY_CREATED, (long)(timestamp / 1000)));
-            orgInfoEntity.setField( new StringField( PROPERTY_NAME, name ));
-            orgInfoEntity.setField( new UUIDField( PROPERTY_UUID, orgUuid ));
-
-            EntityCollectionManager ecm = getManagerCache()
-                    .getEntityCollectionManager( SYSTEM_ORGS_SCOPE );
-            EntityIndex eci = getManagerCache()
-                    .getEntityIndex( SYSTEM_ORGS_INDEX_SCOPE );
-
-            orgInfoEntity = ecm.write( orgInfoEntity ).toBlockingObservable().last();
-            eci.index( orgInfoEntity );
-            eci.refresh();
+            // create new org because the specified one does not exist
+            final String orgName = organizationName;
+            Entity orgInfo = em.create("organization", new HashMap<String, Object>() {{
+                put( PROPERTY_NAME, orgName );
+            }});
+            em.refreshIndex();
+            orgUuid = orgInfo.getUuid();
         }
 
+        // create appinfo entry in the system app
+        final UUID appId = applicationId;
+        final UUID orgId = orgUuid;
+        Map<String, Object> appInfoMap = new HashMap<String, Object>() {{
+            put( PROPERTY_NAME, appName );
+            put( "applicationUuid", appId );
+            put( "organizationUuid", orgId );
+        }};
+        Entity appInfo = em.create( "appinfo", appInfoMap );
+        em.refreshIndex();
+
+        // create application entity
         if ( properties == null ) {
             properties = new TreeMap<String, Object>( CASE_INSENSITIVE_ORDER );
         }
         properties.put( PROPERTY_NAME, appName );
+        EntityManager appEm = getEntityManager( applicationId );
+        appEm.create( applicationId, TYPE_APPLICATION, properties );
+        appEm.resetRoles();
+        appEm.refreshIndex();
 
-        Entity appInfoEntity = new Entity( generateApplicationId( applicationId ));
-
-        long timestamp = System.currentTimeMillis();
-        appInfoEntity.setField( new LongField( PROPERTY_CREATED, (long)(timestamp / 1000)));
-        appInfoEntity.setField( new StringField( PROPERTY_NAME, name ));
-        appInfoEntity.setField( new UUIDField( "applicationUuid", applicationId ));
-        appInfoEntity.setField( new UUIDField( "organizationUuid", orgUuid ));
-
-        // create app in system app scope
-        {
-            EntityCollectionManager ecm = getManagerCache()
-                    .getEntityCollectionManager(SYSTEM_APPS_SCOPE );
-            EntityIndex eci = getManagerCache()
-                    .getEntityIndex( SYSTEM_APPS_INDEX_SCOPE );
-
-            appInfoEntity = ecm.write( appInfoEntity ).toBlockingObservable().last();
-            eci.index( appInfoEntity );
-            eci.refresh();
-        }
-
-        // create app in its own scope
-        EntityManager em = getEntityManager( applicationId );
-        em.create( applicationId, TYPE_APPLICATION, properties );
-        em.resetRoles();
-        em.refreshIndex();
-
+        logger.info("Initialized application {}", appName );
         return applicationId;
     }
 
@@ -337,8 +282,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
     public ApplicationScope getApplicationScope( UUID applicationId ) {
 
         // We can always generate a scope, it doesn't matter if  the application exists yet or not.
-
-        final ApplicationScopeImpl scope = new ApplicationScopeImpl( generateApplicationId( applicationId ) );
+        final ApplicationScopeImpl scope = 
+                new ApplicationScopeImpl( generateApplicationId( applicationId ) );
 
         return scope;
     }
@@ -348,39 +293,40 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
     public UUID importApplication(
             String organization, UUID applicationId,
             String name, Map<String, Object> properties) throws Exception {
+
         throw new UnsupportedOperationException("Not supported yet.");
     }
 
     
     public UUID lookupOrganization( String name) throws Exception {
+        init();
 
         Query q = Query.fromQL(PROPERTY_NAME + " = '" + name + "'");
-
-        EntityIndex ei = getManagerCache().getEntityIndex( SYSTEM_ORGS_INDEX_SCOPE );
-        CandidateResults results = ei.search( q );
+        EntityManager em = getEntityManager(SYSTEM_APP_ID);
+        Results results = em.searchCollection( em.getApplicationRef(), "organizations", q);
 
         if ( results.isEmpty() ) {
             return null; 
         } 
 
-        return results.iterator().next().getId().getUuid();
+        return results.iterator().next().getUuid();
     }
 
 
     @Override
-    public UUID lookupApplication( String name) throws Exception {
+    public UUID lookupApplication( String name ) throws Exception {
+        init();
 
         Query q = Query.fromQL( PROPERTY_NAME + " = '" + name + "'");
 
-        EntityIndex ei = getManagerCache().getEntityIndex( SYSTEM_APPS_INDEX_SCOPE );
-        
-        CandidateResults results = ei.search( q );
+        EntityManager em = getEntityManager(SYSTEM_APP_ID);
+        Results results = em.searchCollection( em.getApplicationRef(), "appinfos", q);
 
         if ( results.isEmpty() ) {
             return null; 
         } 
 
-        return results.iterator().next().getId().getUuid();
+        return results.iterator().next().getUuid();
     }
 
 
@@ -388,45 +334,47 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
     @Metered(group = "core", name = "EntityManagerFactory_getApplication")
     public Map<String, UUID> getApplications() throws Exception {
 
-        EntityCollectionManager em = getManagerCache()
-                .getEntityCollectionManager(SYSTEM_APPS_SCOPE );
-        EntityIndex ei = getManagerCache()
-                .getEntityIndex( SYSTEM_APPS_INDEX_SCOPE );
-
         Map<String, UUID> appMap = new HashMap<String, UUID>();
 
-        String cursor = null;
-        boolean done = false;
+        ApplicationScope appScope = getApplicationScope(SYSTEM_APP_ID);
+        GraphManager gm = managerCache.getGraphManager(appScope);
 
-        while ( !done ) {
+        EntityManager em = getEntityManager(SYSTEM_APP_ID);
+        Application app = em.getApplication();
+        Id fromEntityId = new SimpleId( app.getUuid(), app.getType() );
 
-            Query q = Query.fromQL("select *");
-            q.setCursor( cursor );
+        String edgeType = CpEntityManager.getEdgeTypeFromCollectionName("appinfos", "appinfo");
 
-            CandidateResults results = ei.search( q );
-            cursor = results.getCursor();
+        logger.debug("getApplications(): Loading edges of edgeType {} from {}:{}", 
+            new Object[] { edgeType, fromEntityId.getType(), fromEntityId.getUuid() } );
 
-            Iterator<CandidateResult> iter = results.iterator();
-            while ( iter.hasNext() ) {
+        Observable<Edge> edges = gm.loadEdgesFromSource( new SimpleSearchByEdgeType( 
+                fromEntityId, edgeType, Long.MAX_VALUE, 
+                SearchByEdgeType.Order.DESCENDING, null ));
+        
+        Iterator<Edge> iter = edges.toBlockingObservable().getIterator();
+        while ( iter.hasNext() ) {
 
-                CandidateResult cr = iter.next();
-                Entity e = em.load( cr.getId() ).toBlockingObservable().last();
+            Edge edge = iter.next();
+            Id targetId = edge.getTargetNode();
 
-                if ( cr.getVersion().compareTo( e.getVersion()) < 0 )  {
-                    logger.debug("Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}", 
-                        new Object[] { cr.getId().getUuid(), cr.getId().getType(), 
-                            cr.getVersion(), e.getVersion()});
-                    continue;
-                }
-                
-                appMap.put( 
-                    (String)e.getField(PROPERTY_NAME).getValue(), 
-                    (UUID)e.getField("applicationUuid").getValue() );
-            }
+            logger.debug("getApplications(): Processing edge from {}:{} to {}:{}", new Object[] {
+                edge.getSourceNode().getType(), edge.getSourceNode().getUuid(), 
+                edge.getTargetNode().getType(), edge.getTargetNode().getUuid() 
+            });
 
-            if ( cursor == null ) {
-                done = true;
-            }
+            CollectionScope collScope = new CollectionScopeImpl(
+                    appScope.getApplication(),
+                    appScope.getApplication(),
+                    CpEntityManager.getCollectionScopeNameFromCollectionName("appinfos"));
+
+            org.apache.usergrid.persistence.model.entity.Entity e = 
+                    managerCache.getEntityCollectionManager( collScope ).load( targetId )
+                        .toBlockingObservable().lastOrDefault(null);
+
+            appMap.put( 
+                (String)e.getField( PROPERTY_NAME ).getValue(), 
+                (UUID)e.getField( "applicationUuid" ).getValue());
         }
 
         return appMap;
@@ -442,30 +390,26 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
     @Override
     public Map<String, String> getServiceProperties() {
 
-        EntityIndex ei = getManagerCache()
-                .getEntityIndex( SYSTEM_PROPS_INDEX_SCOPE );
-        EntityCollectionManager em = getManagerCache()
-                .getEntityCollectionManager( SYSTEM_PROPS_SCOPE );
+        Map<String, String> props = new HashMap<String,String>();
 
+        EntityManager em = getEntityManager(SYSTEM_APP_ID);
         Query q = Query.fromQL("select *");
+        Results results = null;
+        try {
+            results = em.searchCollection( em.getApplicationRef(), "propertymaps", q);
 
-        CandidateResults results = ei.search( q );
-
-        if ( results.isEmpty() ) {
-            return new HashMap<String,String>();
+        } catch (Exception ex) {
+            logger.error("Error getting system properties", ex);
         }
 
-        CandidateResult cr = results.iterator().next();
-        Entity propsEntity = em.load( cr.getId() ).toBlockingObservable().last();
-
-        Map<String, String> props = new HashMap<String, String>();
-
-        // intentionally going only one-level deep into fields and treating all 
-        // values as strings because that is all we need for service properties.
-        for ( Field f : propsEntity.getFields() ) {
-            props.put( f.getName(), f.getValue().toString() ); 
+        if ( results == null || results.isEmpty() ) {
+            return props;
         }
 
+        org.apache.usergrid.persistence.Entity e = results.getEntity();
+        for ( String key : e.getProperties().keySet() ) {
+            props.put( key, props.get(key).toString() );
+        }
         return props;
     }
 
@@ -473,30 +417,39 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
     @Override
     public boolean updateServiceProperties(Map<String, String> properties) {
 
-        EntityCollectionManager em = getManagerCache()
-            .getEntityCollectionManager( SYSTEM_PROPS_SCOPE );
-        EntityIndex ei = getManagerCache()
-            .getEntityIndex( SYSTEM_PROPS_INDEX_SCOPE );
-
+        EntityManager em = getEntityManager(SYSTEM_APP_ID);
         Query q = Query.fromQL("select *");
-        CandidateResults results = ei.search( q );
-        Entity propsEntity;
+        Results results = null;
+        try {
+            results = em.searchCollection( em.getApplicationRef(), "propertymaps", q);
+
+        } catch (Exception ex) {
+            logger.error("Error getting system properties", ex);
+            return false;
+        }
+
+        org.apache.usergrid.persistence.Entity propsEntity = null;
+
         if ( !results.isEmpty() ) {
-            propsEntity = em.load( results.iterator().next().getId()).toBlockingObservable().last();
+            propsEntity = results.getEntity();
+
         } else {
-            propsEntity = new Entity( new SimpleId( "properties" ));
-            long timestamp = System.currentTimeMillis();
-            propsEntity.setField( new LongField( PROPERTY_CREATED, (long)(timestamp / 1000)));
+            propsEntity = EntityFactory.newEntity( UUIDUtils.newTimeUUID(), "propertymap");
         }
 
         // intentionally going only one-level deep into fields and treating all 
         // values as strings because that is all we need for service properties
         for ( String key : properties.keySet() ) {
-            propsEntity.setField( new StringField(key, properties.get(key)) );
+            propsEntity.setProperty( key, properties.get(key).toString() );
         }
 
-        propsEntity = em.write( propsEntity ).toBlockingObservable().last();
-        ei.index( propsEntity );    
+        try {
+            em.update( propsEntity );
+
+        } catch (Exception ex) {
+            logger.error("Error updating service properties", ex);
+            return false;
+        }
 
         return true;
     }
@@ -513,27 +466,34 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
     @Override
     public boolean deleteServiceProperty(String name) {
 
-        EntityCollectionManager em = getManagerCache().getEntityCollectionManager( SYSTEM_PROPS_SCOPE );
-        EntityIndex ei = getManagerCache().getEntityIndex( SYSTEM_PROPS_INDEX_SCOPE );
-
+        EntityManager em = getEntityManager(SYSTEM_APP_ID);
         Query q = Query.fromQL("select *");
-        CandidateResults results = ei.search( q );
-
-        Entity propsEntity = em.load( 
-                results.iterator().next().getId() ).toBlockingObservable().last();
+        Results results = null;
+        try {
+            results = em.searchCollection( em.getApplicationRef(), "propertymaps", q);
 
-        if ( propsEntity == null ) {
-            return false; // nothing to delete
+        } catch (Exception ex) {
+            logger.error("Error getting service property for delete of property: " + name, ex);
+            return false;
         }
 
-        if ( propsEntity.getField(name) == null ) {
-            return false; // no such field
+        org.apache.usergrid.persistence.Entity propsEntity = null;
+
+        if ( !results.isEmpty() ) {
+            propsEntity = results.getEntity();
+
+        } else {
+            propsEntity = EntityFactory.newEntity( UUIDUtils.newTimeUUID(), "propertymap");
         }
 
-        propsEntity.removeField( name );
+        try {
+            ((AbstractEntity)propsEntity).clearDataset( name );
+            em.update( propsEntity );
 
-        propsEntity = em.write( propsEntity ).toBlockingObservable().last();
-        ei.index( propsEntity );    
+        } catch (Exception ex) {
+            logger.error("Error deleting service property name: " + name, ex);
+            return false;
+        }
 
         return true;
     }
@@ -565,15 +525,12 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
         return MANAGEMENT_APPLICATION_ID;
     }
 
+
     @Override
     public UUID getDefaultAppId() {
         return DEFAULT_APPLICATION_ID; 
     }
 
-    private Id generateOrgId(UUID id){
-        return new SimpleId( id, "organization" );
-    }
-
 
     private Id generateApplicationId(UUID id){
         return new SimpleId( id, Application.ENTITY_TYPE );
@@ -593,19 +550,21 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
 
     public void refreshIndex() {
 
-        // refresh factory's indexes, will refresh all three index scopes
-        managerCache.getEntityIndex( CpEntityManagerFactory.SYSTEM_APPS_INDEX_SCOPE ).refresh();
-
-        // these are unecessary because of above call
-        //managerCache.getEntityIndex( CpEntityManagerFactory.SYSTEM_ORGS_INDEX_SCOPE ).refresh();
-        //managerCache.getEntityIndex( CpEntityManagerFactory.SYSTEM_PROPS_INDEX_SCOPE ).refresh();
-
         // refresh special indexes without calling EntityManager refresh because stack overflow 
+       
+        // system app
+        IndexScope sscope = new IndexScopeImpl( 
+            new SimpleId( SYSTEM_APP_ID, "application"), 
+            new SimpleId( SYSTEM_APP_ID, "application"), "dummy");
+        managerCache.getEntityIndex( sscope ).refresh();
+       
+        // default app
         IndexScope mscope = new IndexScopeImpl( 
             new SimpleId( getManagementAppId(), "application"), 
             new SimpleId( getManagementAppId(), "application"), "dummy");
         managerCache.getEntityIndex( mscope ).refresh();
 
+        // management app
         IndexScope dscope = new IndexScopeImpl( 
             new SimpleId( getDefaultAppId(), "application"), 
             new SimpleId( getDefaultAppId(), "application"), "dummy");
@@ -613,137 +572,36 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
     }
 
 
-    public void rebuildInternalIndexes( ProgressObserver po ) throws Exception {
-
-        // get all connections from systems app
-//        GraphManager gm = managerCache.getGraphManager( CpEntityManagerFactory.SYSTEM_APPS_SCOPE );
-//
-//        Observable<String> edgeTypes = gm.getEdgeTypesFromSource( 
-//            new SimpleSearchEdgeType( systemAppId, null , null ));
-
-        logger.info("Rebuilding system apps index");
-        rebuildIndexScope(
-                CpEntityManagerFactory.SYSTEM_APPS_SCOPE, 
-                CpEntityManagerFactory.SYSTEM_APPS_INDEX_SCOPE, po );
+    public void rebuildAllIndexes( ProgressObserver po ) throws Exception {
 
-        logger.info("Rebuilding system orgs index");
-        rebuildIndexScope(
-                CpEntityManagerFactory.SYSTEM_ORGS_SCOPE,
-                CpEntityManagerFactory.SYSTEM_ORGS_INDEX_SCOPE, po );
+        logger.info("\n\nRebuilding all indexes\n");
 
-        logger.info("Rebuilding system props index");
-        rebuildIndexScope(
-                CpEntityManagerFactory.SYSTEM_PROPS_SCOPE,
-                CpEntityManagerFactory.SYSTEM_PROPS_INDEX_SCOPE, po );
+        rebuildInternalIndexes( po );
 
-        logger.info("Rebuilding management application index");
-        rebuildApplicationIndex( MANAGEMENT_APPLICATION_ID, po );
+        Map<String, UUID> appMap = getApplications();
 
-        logger.info("Rebuilding default application index");
-        rebuildApplicationIndex( DEFAULT_APPLICATION_ID, po );
-    }
-
-
-    private void rebuildIndexScope( CollectionScope cs, IndexScope is, ProgressObserver po ) {
-
-        logger.info("Rebuild index scope for {}:{}:{}", new Object[] {
-            cs.getOwner(), cs.getApplication(), cs.getName()
-        });
+        logger.info("About to rebuild indexes for {} applications", appMap.keySet().size());
 
-        EntityCollectionManager ecm = managerCache.getEntityCollectionManager( cs );
-        EntityIndex ei = managerCache.getEntityIndex( is );
-
-        Query q = Query.fromQL("select *");
-        CandidateResults results = ei.search( q );
-
-        Iterator<CandidateResult> iter = results.iterator();
-        while (iter.hasNext()) {
-            CandidateResult cr = iter.next();
-
-            Entity entity = ecm.load(cr.getId()).toBlockingObservable().last();
-
-            if ( cr.getVersion().compareTo( entity.getVersion()) < 0 ) {
-                logger.warn("   Ignoring stale version uuid:{} type:{} state v:{} latest v:{}",
-                    new Object[] { 
-                        cr.getId().getUuid(), cr.getId().getType(), 
-                        cr.getVersion(), entity.getVersion()
-                    });
-
-            } else {
-
-                logger.info("   Updating entity type {} with id {} for app {}/{}", new Object[] { 
-                    cr.getId().getType(), cr.getId().getUuid(), cs.getApplication().getUuid()
-                });
-
-                ei.index(entity);
-
-                if ( po != null ) {
-                    po.onProgress();
-                }
-
-            }
+        for ( UUID appUuid : appMap.values() ) {
+            rebuildApplicationIndexes( appUuid, po );
         }
     }
+   
 
-
-    public void rebuildApplicationIndex( UUID appId, ProgressObserver po ) throws Exception {
-
-        EntityManager em = getEntityManager( appId );
-
-        Set<String> collections = em.getApplicationCollections();
-
-        logger.debug("For app {} found {} collections: {}", new Object[] {
-            appId, collections.size(), collections });
-
-        for ( String collection : collections ) {
-            rebuildCollectionIndex( appId, collection, po );
-        }
+    @Override
+    public void rebuildInternalIndexes(ProgressObserver po) throws Exception {
+        rebuildApplicationIndexes(SYSTEM_APP_ID, po);
     }
 
 
-    public void rebuildCollectionIndex( UUID appId, String collectionName, ProgressObserver po ) 
-            throws Exception {
-
-        logger.info( "Reindexing collection: {} for app id: {}", collectionName, appId );
-
+    @Override
+    public void rebuildApplicationIndexes( UUID appId, ProgressObserver po ) throws Exception {
+        
+        logger.info("Rebuilding index for application id {}", appId);
         EntityManager em = getEntityManager( appId );
         Application app = em.getApplication();
 
-        // search for all orgs
-
-        Query query = new Query();
-        query.setLimit(REBUILD_PAGE_SIZE );
-        Results r = null;
-
-        do {
-
-            r = em.searchCollection( app, collectionName, query );
-
-            for ( org.apache.usergrid.persistence.Entity entity : r.getEntities() ) {
-
-                logger.info( "   Updating Entity name {}, type: {}, id: {} in app id: {}", new Object[] {
-                        entity.getName(), entity.getType(), entity.getUuid(), appId
-                } );
-
-                try {
-                    em.update( entity );
-
-                    if ( po != null ) {
-                        po.onProgress();
-                    }
-                }
-                catch ( DuplicateUniquePropertyExistsException dupee ) {
-                    logger.error( "   Duplicate property for type: {} with id: {} for app id: {}.  "
-                            + "Property name: {} , value: {}", new Object[] {
-                            entity.getType(), entity.getUuid(), appId, dupee.getPropertyName(), 
-                            dupee.getPropertyValue()
-                    } );
-                }
-            }
-
-            query.setCursor( r.getCursor() );
-        }
-        while ( r != null && r.size() == REBUILD_PAGE_SIZE );
+        ((CpEntityManager)em).reindex( po );
     }
 
 
@@ -756,142 +614,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
         }
     }
 
-
-    private void populateSystemOrgsFromEs() throws Exception {
-
-        logger.info("Migrating system orgs");
-
-        EntityCollectionManager ecm = getManagerCache()
-                .getEntityCollectionManager(SYSTEM_ORGS_SCOPE);
-        EntityIndex ei = getManagerCache()
-                .getEntityIndex( SYSTEM_ORGS_INDEX_SCOPE );
-
-        EntityManager systemAppEm = getEntityManager(SYSTEM_APP_ID);
-
-        String cursor = null;
-        boolean done = false;
-
-        while ( !done ) {
-
-            Query q = Query.fromQL("select *");
-            q.setCursor( cursor );
-
-            CandidateResults results = ei.search( q );
-            cursor = results.getCursor();
-
-            Iterator<CandidateResult> iter = results.iterator();
-            while ( iter.hasNext() ) {
-
-                CandidateResult cr = iter.next();
-                Entity e = ecm.load( cr.getId() ).toBlockingObservable().last();
-
-                if ( cr.getVersion().compareTo( e.getVersion()) < 0 )  {
-                    logger.debug("Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}", 
-                        new Object[] { cr.getId().getUuid(), cr.getId().getType(), 
-                            cr.getVersion(), e.getVersion()});
-                    continue;
-                }
-
-                Map<String, Object> entityMap = CpEntityMapUtils.toMap( e );
-                systemAppEm.create("organization", entityMap );
-            }
-
-            if ( cursor == null ) {
-                done = true;
-            }
-        }
-    }
-
-
-    private void populateSystemAppsFromEs() throws Exception {
-
-        logger.info("Migrating system apps");
-
-        EntityCollectionManager ecm = getManagerCache()
-                .getEntityCollectionManager(SYSTEM_APPS_SCOPE );
-        EntityIndex ei = getManagerCache()
-                .getEntityIndex( SYSTEM_APPS_INDEX_SCOPE );
-
-        EntityManager systemAppEm = getEntityManager(SYSTEM_APP_ID);
-
-        String cursor = null;
-        boolean done = false;
-
-        while ( !done ) {
-
-            Query q = Query.fromQL("select *");
-            q.setCursor( cursor );
-
-            CandidateResults results = ei.search( q );
-            cursor = results.getCursor();
-
-            Iterator<CandidateResult> iter = results.iterator();
-            while ( iter.hasNext() ) {
-
-                CandidateResult cr = iter.next();
-                Entity e = ecm.load( cr.getId() ).toBlockingObservable().last();
-
-                if ( cr.getVersion().compareTo( e.getVersion()) < 0 )  {
-                    logger.debug("Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}", 
-                        new Object[] { cr.getId().getUuid(), cr.getId().getType(), 
-                            cr.getVersion(), e.getVersion()});
-                    continue;
-                }
-
-                Map<String, Object> entityMap = CpEntityMapUtils.toMap( e );
-                systemAppEm.create("application", entityMap );
-            }
-
-            if ( cursor == null ) {
-                done = true;
-            }
-        }
-    }
-
-
-    private void populateSystemPropsFromEs() throws Exception {
-
-        logger.info("Migrating system props");
-
-        EntityCollectionManager ecm = getManagerCache()
-                .getEntityCollectionManager(SYSTEM_PROPS_SCOPE );
-        EntityIndex ei = getManagerCache()
-                .getEntityIndex( SYSTEM_PROPS_INDEX_SCOPE );
-
-        EntityManager systemAppEm = getEntityManager(SYSTEM_APP_ID);
-
-        String cursor = null;
-        boolean done = false;
-
-        while ( !done ) {
-
-            Query q = Query.fromQL("select *");
-            q.setCursor( cursor );
-
-            CandidateResults results = ei.search( q );
-            cursor = results.getCursor();
-
-            Iterator<CandidateResult> iter = results.iterator();
-            while ( iter.hasNext() ) {
-
-                CandidateResult cr = iter.next();
-                Entity e = ecm.load( cr.getId() ).toBlockingObservable().last();
-
-                if ( cr.getVersion().compareTo( e.getVersion()) < 0 )  {
-                    logger.debug("Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}", 
-                        new Object[] { cr.getId().getUuid(), cr.getId().getType(), 
-                            cr.getVersion(), e.getVersion()});
-                    continue;
-                }
-
-                Map<String, Object> entityMap = CpEntityMapUtils.toMap( e );
-                systemAppEm.create("property", entityMap );
-            }
-
-            if ( cursor == null ) {
-                done = true;
-            }
-        }
+    @Override
+    public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver po ) {
     }
-
 }


Mime
View raw message