Return-Path: X-Original-To: apmail-usergrid-commits-archive@minotaur.apache.org Delivered-To: apmail-usergrid-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 3010F17554 for ; Thu, 2 Oct 2014 07:32:33 +0000 (UTC) Received: (qmail 7272 invoked by uid 500); 2 Oct 2014 07:32:33 -0000 Delivered-To: apmail-usergrid-commits-archive@usergrid.apache.org Received: (qmail 7216 invoked by uid 500); 2 Oct 2014 07:32:33 -0000 Mailing-List: contact commits-help@usergrid.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@usergrid.incubator.apache.org Delivered-To: mailing list commits@usergrid.incubator.apache.org Received: (qmail 7131 invoked by uid 99); 2 Oct 2014 07:32:33 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 02 Oct 2014 07:32:32 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id A7549A0BA47; Thu, 2 Oct 2014 07:32:32 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: toddnine@apache.org To: commits@usergrid.apache.org Date: Thu, 02 Oct 2014 07:32:39 -0000 Message-Id: <6058c48fb03d4776b99896f768280a1e@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [08/13] git commit: Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into esbatching Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into esbatching Conflicts: stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ae9974d3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ae9974d3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ae9974d3 Branch: refs/heads/two-dot-o Commit: ae9974d38210cdef353820d40d8a59d0f0f4140b Parents: bba08dd b2b8886 Author: Todd Nine Authored: Wed Oct 1 23:42:59 2014 -0600 Committer: Todd Nine Committed: Wed Oct 1 23:42:59 2014 -0600 ---------------------------------------------------------------------- .../corepersistence/CpEntityManager.java | 528 +++++++++++++++---- .../corepersistence/CpEntityManagerFactory.java | 499 +++++++----------- .../corepersistence/CpRelationManager.java | 341 +++++------- .../usergrid/corepersistence/CpSetup.java | 55 +- .../HybridEntityManagerFactory.java | 18 +- .../persistence/EntityManagerFactory.java | 12 +- .../persistence/cassandra/CassandraService.java | 5 - .../cassandra/EntityManagerFactoryImpl.java | 54 +- .../PerformanceEntityRebuildIndexTest.java | 278 ++++++++++ .../cassandra/EntityManagerFactoryImplIT.java | 10 +- stack/core/src/test/resources/log4j.properties | 13 +- .../usergrid/persistence/index/IndexFig.java | 7 + .../index/impl/EsEntityIndexImpl.java | 18 + .../persistence/index/impl/EsProvider.java | 15 + .../org/apache/usergrid/tools/IndexRebuild.java | 63 ++- stack/tools/src/main/resources/log4j.properties | 18 +- 16 files changed, 1202 insertions(+), 732 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ae9974d3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java ---------------------------------------------------------------------- diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java index a5475e7,38453e1..5ce871b --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java @@@ -73,8 -121,12 +121,13 @@@ import org.apache.usergrid.persistence. import org.apache.usergrid.persistence.exceptions.EntityNotFoundException; import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException; import org.apache.usergrid.persistence.exceptions.UnexpectedEntityTypeException; + import org.apache.usergrid.persistence.graph.Edge; + import org.apache.usergrid.persistence.graph.GraphManager; + import org.apache.usergrid.persistence.graph.SearchByEdgeType; + import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType; + import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType; import org.apache.usergrid.persistence.index.EntityIndex; +import org.apache.usergrid.persistence.index.EntityIndexBatch; import org.apache.usergrid.persistence.index.IndexScope; import org.apache.usergrid.persistence.index.impl.IndexScopeImpl; import org.apache.usergrid.persistence.index.query.CounterResolution; @@@ -586,24 -644,20 +650,24 @@@ public class CpEntityManager implement } } + + // deindex from default index scope IndexScope defaultIndexScope = new IndexScopeImpl( - appScope.getApplication(), - appScope.getApplication(), + applicationScope.getApplication(), + applicationScope.getApplication(), getCollectionScopeNameFromEntityType( entityRef.getType() ) ); - EntityIndex entityIndex = managerCache.getEntityIndex( defaultIndexScope ); - entityIndex.deindex( entity ); + + batch.deindex(defaultIndexScope, entity ); IndexScope allTypesIndexScope = new IndexScopeImpl( - appScope.getApplication(), - appScope.getApplication(), + applicationScope.getApplication(), + applicationScope.getApplication(), ALL_TYPES); - EntityIndex aei = managerCache.getEntityIndex( allTypesIndexScope ); - aei.deindex( entity ); + + batch.deindex( allTypesIndexScope, entity ); + + batch.execute(); decrementEntityCollection( Schema.defaultCollectionName( entityId.getType() ) ); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ae9974d3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java ---------------------------------------------------------------------- diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java index 090a4d6,accf2f8..9a10afd --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java @@@ -47,23 -46,18 +46,19 @@@ import org.apache.usergrid.persistence. import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl; import org.apache.usergrid.persistence.entities.Application; import org.apache.usergrid.persistence.exceptions.ApplicationAlreadyExistsException; - import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException; + import org.apache.usergrid.persistence.graph.Edge; + import org.apache.usergrid.persistence.graph.GraphManager; import org.apache.usergrid.persistence.graph.GraphManagerFactory; + import org.apache.usergrid.persistence.graph.SearchByEdgeType; + import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType; +import org.apache.usergrid.persistence.index.EntityIndex; - import org.apache.usergrid.persistence.index.EntityIndexBatch; import org.apache.usergrid.persistence.index.EntityIndexFactory; import org.apache.usergrid.persistence.index.IndexScope; - import org.apache.usergrid.persistence.index.query.CandidateResult; import org.apache.usergrid.persistence.index.impl.IndexScopeImpl; - import org.apache.usergrid.persistence.index.query.CandidateResults; import org.apache.usergrid.persistence.index.query.Query; - import org.apache.usergrid.persistence.model.entity.Entity; import org.apache.usergrid.persistence.model.entity.Id; import org.apache.usergrid.persistence.model.entity.SimpleId; - import org.apache.usergrid.persistence.model.field.Field; - import org.apache.usergrid.persistence.model.field.LongField; - import org.apache.usergrid.persistence.model.field.StringField; - import org.apache.usergrid.persistence.model.field.UUIDField; + import org.apache.usergrid.utils.UUIDUtils; import org.apache.usergrid.persistence.model.util.UUIDGenerator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@@ -321,11 -299,11 +300,11 @@@ public class CpEntityManagerFactory imp public UUID lookupOrganization( String name) throws Exception { + init(); Query q = Query.fromQL(PROPERTY_NAME + " = '" + name + "'"); - - EntityIndex ei = getManagerCache().getEntityIndex( SYSTEM_ORGS_INDEX_SCOPE ); - CandidateResults results = ei.search(SYSTEM_ORGS_INDEX_SCOPE, q ); - EntityManager em = getEntityManager(SYSTEM_APP_ID); - Results results = em.searchCollection( em.getApplicationRef(), "organizations", q); ++ EntityManager em = getEntityManager( SYSTEM_APP_ID ); ++ Results results = em.searchCollection( em.getApplicationRef(), "organizations", q ); if ( results.isEmpty() ) { return null; http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ae9974d3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java ---------------------------------------------------------------------- diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java index 260e429,fba758f..0073eb1 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java @@@ -27,13 -29,13 +29,12 @@@ import java.util.List import java.util.Map; import java.util.Set; import java.util.UUID; -import me.prettyprint.hector.api.Keyspace; -import me.prettyprint.hector.api.beans.DynamicComposite; -import me.prettyprint.hector.api.beans.HColumn; -import static me.prettyprint.hector.api.factory.HFactory.createMutator; -import me.prettyprint.hector.api.mutation.Mutator; -import static org.apache.usergrid.corepersistence.CpEntityManager.getEdgeTypeFromCollectionName; -import static org.apache.usergrid.corepersistence.CpEntityManager.getEdgeTypeFromConnectionType; + - import org.apache.usergrid.persistence.index.EntityIndexBatch; +import org.apache.usergrid.utils.UUIDUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.Assert; + import org.apache.usergrid.persistence.ConnectedEntityRef; import org.apache.usergrid.persistence.ConnectionRef; import org.apache.usergrid.persistence.Entity; @@@ -1108,25 -998,13 +1009,25 @@@ public class CpRelationManager implemen GraphManager gm = managerCache.getGraphManager(applicationScope); gm.writeEdge(edge).toBlockingObservable().last(); - final EntityIndex ei = managerCache.getEntityIndex(applicationScope); - final EntityIndexBatch batch = ei.createBatch(); - - logger.debug("Wrote edgeType {}\n from {}:{}\n to {}:{}\n scope {}:{}", new Object[] { - edgeType, cpHeadEntity.getId().getType(), cpHeadEntity.getId().getUuid(), - targetEntity.getId().getType(), targetEntity.getId().getUuid(), - applicationScope.getApplication().getType(), applicationScope.getApplication().getUuid()}); + // Index the new connection in app|source|type context + IndexScope indexScope = new IndexScopeImpl( + applicationScope.getApplication(), + cpHeadEntity.getId(), + CpEntityManager.getConnectionScopeName( connectedEntityRef.getType(), connectionType )); - batch.index( indexScope, targetEntity ); ++ EntityIndex ei = managerCache.getEntityIndex(indexScope); ++ ei.index( targetEntity ); + + // Index the new connection in app|scope|all-types context + IndexScope allTypesIndexScope = new IndexScopeImpl( + applicationScope.getApplication(), + cpHeadEntity.getId(), + ALL_TYPES); ++ EntityIndex aei = managerCache.getEntityIndex(allTypesIndexScope); ++ aei.index( targetEntity ); - ((CpEntityManager)em).indexEntityIntoConnection( - cpHeadEntity, targetEntity, connectedEntityRef.getType(), connectionType ); + batch.index( allTypesIndexScope, targetEntity ); + + batch.execute(); Keyspace ko = cass.getApplicationKeyspace( applicationId ); Mutator m = createMutator( ko, be ); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ae9974d3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java index 56316d8,2eb8414..7851afe --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java @@@ -281,18 -674,32 +281,36 @@@ public class EsEntityIndexImpl implemen } - @Override + public void refresh() { + client.admin().indices().prepareRefresh( indexName ).execute().actionGet(); + log.debug("Refreshed index: " + indexName); + } + - @Override - public CandidateResults getEntityVersions(Id id) { + public CandidateResults getEntityVersions( final IndexScope scope, final Id id ) { Query query = new Query(); - query.addEqualityFilter(ENTITYID_FIELDNAME,id.getUuid().toString()); - CandidateResults results = search( query ); + query.addEqualityFilter( ENTITYID_FIELDNAME, id.getUuid().toString() ); + CandidateResults results = search( scope, query ); return results; } + /** + * For testing only. + */ + public void deleteIndex() { + AdminClient adminClient = client.admin(); + DeleteIndexResponse response = adminClient.indices().prepareDelete( indexName ).get(); + if ( response.isAcknowledged() ) { + log.info("Deleted index: " + indexName ); + } else { + log.info("Failed to delete index " + indexName ); + } + } + + + @Override + public void refresh() { + client.admin().indices().prepareRefresh( indexName ).execute().actionGet(); + log.debug( "Refreshed index: " + indexName ); + } }