usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From toddn...@apache.org
Subject [08/13] git commit: Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into esbatching
Date Thu, 02 Oct 2014 07:32:39 GMT
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
esbatching

Conflicts:
	stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
	stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
	stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
	stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ae9974d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ae9974d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ae9974d3

Branch: refs/heads/two-dot-o
Commit: ae9974d38210cdef353820d40d8a59d0f0f4140b
Parents: bba08dd b2b8886
Author: Todd Nine <toddnine@apache.org>
Authored: Wed Oct 1 23:42:59 2014 -0600
Committer: Todd Nine <toddnine@apache.org>
Committed: Wed Oct 1 23:42:59 2014 -0600

----------------------------------------------------------------------
 .../corepersistence/CpEntityManager.java        | 528 +++++++++++++++----
 .../corepersistence/CpEntityManagerFactory.java | 499 +++++++-----------
 .../corepersistence/CpRelationManager.java      | 341 +++++-------
 .../usergrid/corepersistence/CpSetup.java       |  55 +-
 .../HybridEntityManagerFactory.java             |  18 +-
 .../persistence/EntityManagerFactory.java       |  12 +-
 .../persistence/cassandra/CassandraService.java |   5 -
 .../cassandra/EntityManagerFactoryImpl.java     |  54 +-
 .../PerformanceEntityRebuildIndexTest.java      | 278 ++++++++++
 .../cassandra/EntityManagerFactoryImplIT.java   |  10 +-
 stack/core/src/test/resources/log4j.properties  |  13 +-
 .../usergrid/persistence/index/IndexFig.java    |   7 +
 .../index/impl/EsEntityIndexImpl.java           |  18 +
 .../persistence/index/impl/EsProvider.java      |  15 +
 .../org/apache/usergrid/tools/IndexRebuild.java |  63 ++-
 stack/tools/src/main/resources/log4j.properties |  18 +-
 16 files changed, 1202 insertions(+), 732 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ae9974d3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index a5475e7,38453e1..5ce871b
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@@ -73,8 -121,12 +121,13 @@@ import org.apache.usergrid.persistence.
  import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
  import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException;
  import org.apache.usergrid.persistence.exceptions.UnexpectedEntityTypeException;
+ import org.apache.usergrid.persistence.graph.Edge;
+ import org.apache.usergrid.persistence.graph.GraphManager;
+ import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+ import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
+ import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
  import org.apache.usergrid.persistence.index.EntityIndex;
 +import org.apache.usergrid.persistence.index.EntityIndexBatch;
  import org.apache.usergrid.persistence.index.IndexScope;
  import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
  import org.apache.usergrid.persistence.index.query.CounterResolution;
@@@ -586,24 -644,20 +650,24 @@@ public class CpEntityManager implement
                  }
              }
  
 +
 +
              // deindex from default index scope
              IndexScope defaultIndexScope = new IndexScopeImpl( 
-                     appScope.getApplication(), 
-                     appScope.getApplication(),
+                     applicationScope.getApplication(), 
+                     applicationScope.getApplication(),
                      getCollectionScopeNameFromEntityType( entityRef.getType() ) );
 -            EntityIndex entityIndex = managerCache.getEntityIndex( defaultIndexScope );
 -            entityIndex.deindex( entity );
 +
 +            batch.deindex(defaultIndexScope,  entity );
  
              IndexScope allTypesIndexScope = new IndexScopeImpl( 
-                 appScope.getApplication(), 
-                 appScope.getApplication(), 
+                 applicationScope.getApplication(), 
+                 applicationScope.getApplication(), 
                  ALL_TYPES);
 -            EntityIndex aei = managerCache.getEntityIndex( allTypesIndexScope );
 -            aei.deindex( entity );
 +
 +            batch.deindex( allTypesIndexScope,  entity );
 +
 +            batch.execute();
  
              decrementEntityCollection( Schema.defaultCollectionName( entityId.getType()
) );
  

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ae9974d3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 090a4d6,accf2f8..9a10afd
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -47,23 -46,18 +46,19 @@@ import org.apache.usergrid.persistence.
  import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
  import org.apache.usergrid.persistence.entities.Application;
  import org.apache.usergrid.persistence.exceptions.ApplicationAlreadyExistsException;
- import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
+ import org.apache.usergrid.persistence.graph.Edge;
+ import org.apache.usergrid.persistence.graph.GraphManager;
  import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+ import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+ import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
 +import org.apache.usergrid.persistence.index.EntityIndex;
- import org.apache.usergrid.persistence.index.EntityIndexBatch;
  import org.apache.usergrid.persistence.index.EntityIndexFactory;
  import org.apache.usergrid.persistence.index.IndexScope;
- import org.apache.usergrid.persistence.index.query.CandidateResult;
  import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
- import org.apache.usergrid.persistence.index.query.CandidateResults;
  import org.apache.usergrid.persistence.index.query.Query;
- import org.apache.usergrid.persistence.model.entity.Entity;
  import org.apache.usergrid.persistence.model.entity.Id;
  import org.apache.usergrid.persistence.model.entity.SimpleId;
- import org.apache.usergrid.persistence.model.field.Field;
- import org.apache.usergrid.persistence.model.field.LongField;
- import org.apache.usergrid.persistence.model.field.StringField;
- import org.apache.usergrid.persistence.model.field.UUIDField;
+ import org.apache.usergrid.utils.UUIDUtils;
  import org.apache.usergrid.persistence.model.util.UUIDGenerator;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
@@@ -321,11 -299,11 +300,11 @@@ public class CpEntityManagerFactory imp
  
      
      public UUID lookupOrganization( String name) throws Exception {
+         init();
  
          Query q = Query.fromQL(PROPERTY_NAME + " = '" + name + "'");
- 
-         EntityIndex ei = getManagerCache().getEntityIndex( SYSTEM_ORGS_INDEX_SCOPE );
-         CandidateResults results = ei.search(SYSTEM_ORGS_INDEX_SCOPE,  q );
 -        EntityManager em = getEntityManager(SYSTEM_APP_ID);
 -        Results results = em.searchCollection( em.getApplicationRef(), "organizations",
q);
++        EntityManager em = getEntityManager( SYSTEM_APP_ID );
++        Results results = em.searchCollection( em.getApplicationRef(), "organizations",
q );
  
          if ( results.isEmpty() ) {
              return null; 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ae9974d3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 260e429,fba758f..0073eb1
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@@ -27,13 -29,13 +29,12 @@@ import java.util.List
  import java.util.Map;
  import java.util.Set;
  import java.util.UUID;
 -import me.prettyprint.hector.api.Keyspace;
 -import me.prettyprint.hector.api.beans.DynamicComposite;
 -import me.prettyprint.hector.api.beans.HColumn;
 -import static me.prettyprint.hector.api.factory.HFactory.createMutator;
 -import me.prettyprint.hector.api.mutation.Mutator;
 -import static org.apache.usergrid.corepersistence.CpEntityManager.getEdgeTypeFromCollectionName;
 -import static org.apache.usergrid.corepersistence.CpEntityManager.getEdgeTypeFromConnectionType;
 +
- import org.apache.usergrid.persistence.index.EntityIndexBatch;
 +import org.apache.usergrid.utils.UUIDUtils;
 +import org.slf4j.Logger;
 +import org.slf4j.LoggerFactory;
 +import org.springframework.util.Assert;
 +
  import org.apache.usergrid.persistence.ConnectedEntityRef;
  import org.apache.usergrid.persistence.ConnectionRef;
  import org.apache.usergrid.persistence.Entity;
@@@ -1108,25 -998,13 +1009,25 @@@ public class CpRelationManager implemen
          GraphManager gm = managerCache.getGraphManager(applicationScope);
          gm.writeEdge(edge).toBlockingObservable().last();
  
-         final EntityIndex ei = managerCache.getEntityIndex(applicationScope);
-         final EntityIndexBatch batch = ei.createBatch();
- 
 -        logger.debug("Wrote edgeType {}\n   from {}:{}\n   to {}:{}\n   scope {}:{}", new
Object[] { 
 -            edgeType, cpHeadEntity.getId().getType(), cpHeadEntity.getId().getUuid(),
 -            targetEntity.getId().getType(), targetEntity.getId().getUuid(),
 -            applicationScope.getApplication().getType(), applicationScope.getApplication().getUuid()});

 +        // Index the new connection in app|source|type context
 +        IndexScope indexScope = new IndexScopeImpl(
 +            applicationScope.getApplication(), 
 +            cpHeadEntity.getId(), 
 +            CpEntityManager.getConnectionScopeName( connectedEntityRef.getType(), connectionType
));
-        batch.index( indexScope, targetEntity );
++        EntityIndex ei = managerCache.getEntityIndex(indexScope);
++        ei.index( targetEntity );
 +
 +        // Index the new connection in app|scope|all-types context
 +        IndexScope allTypesIndexScope = new IndexScopeImpl(
 +            applicationScope.getApplication(), 
 +            cpHeadEntity.getId(), 
 +            ALL_TYPES);
++        EntityIndex aei = managerCache.getEntityIndex(allTypesIndexScope);
++        aei.index( targetEntity );
  
 -        ((CpEntityManager)em).indexEntityIntoConnection( 
 -            cpHeadEntity, targetEntity, connectedEntityRef.getType(), connectionType );
 +        batch.index( allTypesIndexScope, targetEntity );
 +
 +        batch.execute();
  
          Keyspace ko = cass.getApplicationKeyspace( applicationId );
          Mutator<ByteBuffer> m = createMutator( ko, be );

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ae9974d3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 56316d8,2eb8414..7851afe
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -281,18 -674,32 +281,36 @@@ public class EsEntityIndexImpl implemen
      }
  
  
 -    @Override
+     public void refresh() {
+         client.admin().indices().prepareRefresh( indexName ).execute().actionGet();
+         log.debug("Refreshed index: " + indexName);
+     }
+ 
 -
      @Override
 -    public CandidateResults getEntityVersions(Id id) {
 +    public CandidateResults getEntityVersions( final IndexScope scope, final Id id ) {
          Query query = new Query();
 -        query.addEqualityFilter(ENTITYID_FIELDNAME,id.getUuid().toString());
 -        CandidateResults results = search( query );
 +        query.addEqualityFilter( ENTITYID_FIELDNAME, id.getUuid().toString() );
 +        CandidateResults results = search( scope, query );
          return results;
      }
  
+     /**
+      * For testing only.
+      */
+     public void deleteIndex() {
+         AdminClient adminClient = client.admin();
+         DeleteIndexResponse response = adminClient.indices().prepareDelete( indexName ).get();
+         if ( response.isAcknowledged() ) {
+             log.info("Deleted index: " + indexName );
+         } else {
+             log.info("Failed to delete index " + indexName );
+         }
+     }
+ 
 +
 +    @Override
 +    public void refresh() {
 +        client.admin().indices().prepareRefresh( indexName ).execute().actionGet();
 +        log.debug( "Refreshed index: " + indexName );
 +    }
  }


Mime
View raw message