usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From toddn...@apache.org
Subject [25/50] [abbrv] incubator-usergrid git commit: Fixes stale entity cleanup test to be predictable with sort order.
Date Mon, 17 Nov 2014 20:12:35 GMT
Fixes stale entity cleanup test to be predictable with sort order.

Fixes de-index to pass source scope, rather than trying to determine it.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/75123da3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/75123da3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/75123da3

Branch: refs/heads/two-dot-o-events
Commit: 75123da3c7e53d82c363bb7f790798617023ec8b
Parents: 12b5028
Author: Todd Nine <tnine@apigee.com>
Authored: Wed Nov 12 12:04:22 2014 -0700
Committer: Todd Nine <tnine@apigee.com>
Committed: Wed Nov 12 12:04:22 2014 -0700

----------------------------------------------------------------------
 .../corepersistence/CpRelationManager.java      |  19 ++-
 .../results/FilteringLoader.java                |  26 +--
 .../results/ResultsLoaderFactory.java           |   9 +-
 .../results/ResultsLoaderFactoryImpl.java       |   6 +-
 .../batch/job/AbstractSchedulerRuntimeIT.java   |   2 +
 .../corepersistence/StaleIndexCleanupTest.java  | 163 ++++++++++---------
 .../index/impl/EsEntityIndexBatchImpl.java      |  12 +-
 .../impl/EntityConnectionIndexImplTest.java     | 129 +++++++++++++--
 .../org/apache/usergrid/rest/RestITSuite.java   |   2 +-
 .../org/apache/usergrid/ServiceITSuite.java     |   2 +-
 10 files changed, 235 insertions(+), 135 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75123da3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 9a729b8..f7546b8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -921,6 +921,7 @@ public class CpRelationManager implements RelationManager {
 
         if ( query == null ) {
             query = new Query();
+            query.setCollection( collName );
         }
 
         headEntity = em.validate( headEntity );
@@ -968,11 +969,11 @@ public class CpRelationManager implements RelationManager {
 
             if ( results == null ) {
                 logger.debug( "Calling build results 1" );
-                results = buildResults( query, crs, collName );
+                results = buildResults( indexScope, query, crs, collName );
             }
             else {
                 logger.debug( "Calling build results 2" );
-                Results newResults = buildResults( query, crs, collName );
+                Results newResults = buildResults(indexScope,  query, crs, collName );
                 results.merge( newResults );
             }
 
@@ -1367,7 +1368,7 @@ public class CpRelationManager implements RelationManager {
             query = adjustQuery( query );
             CandidateResults crs = ei.search( indexScope, searchTypes,  query );
 
-            raw = buildResults( query, crs, query.getConnectionType() );
+            raw = buildResults( indexScope, query, crs, query.getConnectionType() );
         }
 
         if ( Level.ALL_PROPERTIES.equals( level ) ) {
@@ -1459,7 +1460,7 @@ public class CpRelationManager implements RelationManager {
         query = adjustQuery( query );
         CandidateResults crs = ei.search( indexScope, searchTypes, query );
 
-        return buildConnectionResults( query, crs, connection );
+        return buildConnectionResults( indexScope, query, crs, connection );
     }
 
 
@@ -1550,11 +1551,11 @@ public class CpRelationManager implements RelationManager {
     }
 
 
-    private Results buildConnectionResults(
-            Query query, CandidateResults crs, String connectionType ) {
+    private Results buildConnectionResults( final IndexScope indexScope,
+            final Query query, final CandidateResults crs, final String connectionType )
{
 
         if ( query.getLevel().equals( Level.ALL_PROPERTIES ) ) {
-            return buildResults( query, crs, connectionType );
+            return buildResults( indexScope, query, crs, connectionType );
         }
 
         final EntityRef sourceRef = new SimpleEntityRef( headEntity.getType(), headEntity.getUuid()
);
@@ -1583,13 +1584,13 @@ public class CpRelationManager implements RelationManager {
      * @param crs Candidates to be considered for results
      * @param collName Name of collection or null if querying all types
      */
-    private Results buildResults( Query query, CandidateResults crs, String collName ) {
+    private Results buildResults( final IndexScope indexScope, final Query query, final CandidateResults
crs, final String collName ) {
 
         logger.debug( "buildResults() for {} from {} candidates", collName, crs.size() );
 
         //get an instance of our results loader
         final ResultsLoader resultsLoader = this.resultsLoaderFactory.getLoader(
-                applicationScope, this.headEntity, query.getResultsLevel() );
+                applicationScope, indexScope, query.getResultsLevel() );
 
         //load the results
         final Results results = resultsLoader.loadResults( crs );

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75123da3/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
index 3c6e96d..d6e63aa 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
@@ -62,24 +62,23 @@ public class FilteringLoader implements ResultsLoader {
 
     private final CpManagerCache managerCache;
     private final ResultsVerifier resultsVerifier;
-    private final Id ownerId;
     private final ApplicationScope applicationScope;
+    private final IndexScope indexScope;
     private final EntityIndexBatch indexBatch;
 
 
     /**
      * Create an instance of a filter loader
      * @param managerCache The manager cache to load
-     * @param resultsVerifier
-     * @param ownerId
-     * @param applicationScope
+     * @param resultsVerifier  The verifier to verify the candidate results
+     * @param applicationScope The application scope to perform the load
+     * @param indexScope The index scope used in the search
      */
-    protected FilteringLoader( final CpManagerCache managerCache, final ResultsVerifier resultsVerifier,
-                               final EntityRef ownerId, final ApplicationScope applicationScope
) {
+    protected FilteringLoader( final CpManagerCache managerCache, final ResultsVerifier resultsVerifier,
 final ApplicationScope applicationScope, final IndexScope indexScope ) {
         this.managerCache = managerCache;
         this.resultsVerifier = resultsVerifier;
-        this.ownerId = new SimpleId( ownerId.getUuid(), ownerId.getType() );
         this.applicationScope = applicationScope;
+        this.indexScope = indexScope;
 
         final EntityIndex index = managerCache.getEntityIndex( applicationScope );
 
@@ -155,7 +154,7 @@ public class FilteringLoader implements ResultsLoader {
 
                 //deindex this document, and remove the previous maxVersion
                 //we have to deindex this from our ownerId, since this is what gave us the
reference
-                deIndex( indexBatch, ownerId, previousMax );
+                indexBatch.deindex( indexScope, previousMax );
                 groupedByScopes.remove( collectionType, previousMax );
 
 
@@ -212,7 +211,7 @@ public class FilteringLoader implements ResultsLoader {
 
                 //ask the loader if this is valid, if not discard it and de-index it
                 if ( !resultsVerifier.isValid( cr ) ) {
-                    deIndex( indexBatch, ownerId, cr );
+                    indexBatch.deindex( indexScope, cr );
                     continue;
                 }
 
@@ -236,13 +235,4 @@ public class FilteringLoader implements ResultsLoader {
     }
 
 
-    protected void deIndex( final EntityIndexBatch batch, final Id ownerId, 
-            final CandidateResult candidateResult ) {
-
-        IndexScope indexScope = new IndexScopeImpl( 
-            ownerId,
-            CpNamingUtils.getCollectionScopeNameFromEntityType( candidateResult.getId().getType())
);
-
-        batch.deindex( indexScope, candidateResult );
-    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75123da3/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java
index 779d604..14fbf88 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java
@@ -22,6 +22,7 @@ package org.apache.usergrid.corepersistence.results;
 
 import org.apache.usergrid.persistence.EntityRef;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.IndexScope;
 import org.apache.usergrid.persistence.index.query.Query;
 
 
@@ -31,8 +32,10 @@ import org.apache.usergrid.persistence.index.query.Query;
 public interface ResultsLoaderFactory {
 
     /**
-     * Get the load for results
+     * Get the loader for results
+     * @param applicationScope The application scope used to load results
+     * @param indexScope The index scope used in the search
+     * @param
      */
-    public ResultsLoader getLoader( final ApplicationScope applicationScope, 
-            final EntityRef ownerId, final Query.Level resultsLevel );
+    public ResultsLoader getLoader( final ApplicationScope applicationScope, final IndexScope
indexScope, final Query.Level resultsLevel );
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75123da3/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactoryImpl.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactoryImpl.java
index 0f39fe3..96928fb 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactoryImpl.java
@@ -23,6 +23,7 @@ package org.apache.usergrid.corepersistence.results;
 import org.apache.usergrid.corepersistence.CpManagerCache;
 import org.apache.usergrid.persistence.EntityRef;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.IndexScope;
 import org.apache.usergrid.persistence.index.query.Query;
 
 import com.google.inject.Inject;
@@ -43,8 +44,7 @@ public class ResultsLoaderFactoryImpl implements ResultsLoaderFactory {
 
 
     @Override
-    public ResultsLoader getLoader( final ApplicationScope applicationScope, 
-            final EntityRef ownerId, final Query.Level resultsLevel ) {
+    public ResultsLoader getLoader( final ApplicationScope applicationScope, final IndexScope
scope, final Query.Level resultsLevel ) {
 
         ResultsVerifier verifier;
 
@@ -58,6 +58,6 @@ public class ResultsLoaderFactoryImpl implements ResultsLoaderFactory {
             verifier = new EntityVerifier(Query.MAX_LIMIT);
         }
 
-        return new FilteringLoader( managerCache, verifier, ownerId, applicationScope );
+        return new FilteringLoader( managerCache, verifier, applicationScope, scope );
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75123da3/stack/core/src/test/java/org/apache/usergrid/batch/job/AbstractSchedulerRuntimeIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/batch/job/AbstractSchedulerRuntimeIT.java
b/stack/core/src/test/java/org/apache/usergrid/batch/job/AbstractSchedulerRuntimeIT.java
index cc31d2d..abf54cb 100644
--- a/stack/core/src/test/java/org/apache/usergrid/batch/job/AbstractSchedulerRuntimeIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/batch/job/AbstractSchedulerRuntimeIT.java
@@ -30,11 +30,13 @@ import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
+import org.junit.Ignore;
 
 
 /**
  * Class to test job runtimes
  */
+@Ignore("These tests no longer work with schared spring context. Need to re-evaluate")
 public class AbstractSchedulerRuntimeIT {
 
     protected static final int DEFAULT_COUNT = 10;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75123da3/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index bbd19ce..fa9f9df 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -15,19 +15,26 @@
  */
 package org.apache.usergrid.corepersistence;
 
-import com.fasterxml.uuid.UUIDComparator;
+
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.commons.lang3.RandomStringUtils;
+
 import org.apache.usergrid.AbstractCoreIT;
 import org.apache.usergrid.corepersistence.util.CpNamingUtils;
 import org.apache.usergrid.persistence.Entity;
 import org.apache.usergrid.persistence.EntityManager;
 import org.apache.usergrid.persistence.EntityRef;
 import org.apache.usergrid.persistence.Results;
-import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
 import org.apache.usergrid.persistence.collection.CollectionScope;
 import org.apache.usergrid.persistence.collection.EntityCollectionManager;
 import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
@@ -42,23 +49,20 @@ import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
 import org.apache.usergrid.persistence.index.query.CandidateResults;
 import org.apache.usergrid.persistence.index.query.Query;
 import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.junit.Assert;
+
+import com.fasterxml.uuid.UUIDComparator;
+
+import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 
 /**
  * Test on read style clean-up of stale ElasticSearch indexes.
  */
 public class StaleIndexCleanupTest extends AbstractCoreIT {
-    private static final Logger logger = LoggerFactory.getLogger(StaleIndexCleanupTest.class
);
+    private static final Logger logger = LoggerFactory.getLogger( StaleIndexCleanupTest.class
);
 
-    // take it easy on embedded Cassandra
-    private static final long writeDelayMs = 50;
-    private static final long readDelayMs = 50;
 
 
     /**
@@ -69,40 +73,38 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
 
         final EntityManager em = app.getEntityManager();
 
-        Entity thing = em.create("thing", new HashMap<String, Object>() {{
-            put("name", "thing1");
-        }});
+        Entity thing = em.create( "thing", new HashMap<String, Object>() {{
+            put( "name", "thing1" );
+        }} );
         em.refreshIndex();
-        
-        assertEquals( 1, queryCollectionCp("things", "thing", "select *").size() );
 
-        org.apache.usergrid.persistence.model.entity.Entity cpEntity = getCpEntity(thing);
+        assertEquals( 1, queryCollectionCp( "things", "thing", "select *" ).size() );
+
+        org.apache.usergrid.persistence.model.entity.Entity cpEntity = getCpEntity( thing
);
         UUID oldVersion = cpEntity.getVersion();
 
-        em.updateProperties(thing, new HashMap<String, Object>() {{
-            put("stuff", "widget");
-        }});
+        em.updateProperties( thing, new HashMap<String, Object>() {{
+            put( "stuff", "widget" );
+        }} );
         em.refreshIndex();
 
-        org.apache.usergrid.persistence.model.entity.Entity cpUpdated = getCpEntity(thing);
-        assertEquals( "widget", cpUpdated.getField("stuff").getValue());
+        org.apache.usergrid.persistence.model.entity.Entity cpUpdated = getCpEntity( thing
);
+        assertEquals( "widget", cpUpdated.getField( "stuff" ).getValue() );
         UUID newVersion = cpUpdated.getVersion();
 
-        assertTrue( "New version is greater than old", 
-                UUIDComparator.staticCompare( newVersion, oldVersion ) > 0 );
+        assertTrue( "New version is greater than old", UUIDComparator.staticCompare( newVersion,
oldVersion ) > 0 );
 
-        assertEquals( 2, queryCollectionCp("things", "thing", "select *").size() );
+        assertEquals( 2, queryCollectionCp( "things", "thing", "select *" ).size() );
     }
 
 
     /**
-     * Test that the CpRelationManager cleans up and stale indexes that it finds when it
is 
-     * building search results.
+     * Test that the CpRelationManager cleans up and stale indexes that it finds when it
is building search results.
      */
     @Test
     public void testStaleIndexCleanup() throws Exception {
 
-        logger.info("Started testStaleIndexCleanup()");
+        logger.info( "Started testStaleIndexCleanup()" );
 
         // TODO: turn off post processing stuff that cleans up stale entities 
 
@@ -111,139 +113,142 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
         final int numEntities = 10;
         final int numUpdates = 3;
 
+        final AtomicInteger updateCount =  new AtomicInteger(  );
+
         // create lots of entities
-        final List<Entity> things = new ArrayList<Entity>(numEntities);
-        for ( int i=0; i<numEntities; i++) {
+        final List<Entity> things = new ArrayList<Entity>( numEntities );
+        for ( int i = 0; i < numEntities; i++ ) {
             final String thingName = "thing" + i;
-            things.add( em.create("thing", new HashMap<String, Object>() {{
-                put("name", thingName);
-            }}));
-            Thread.sleep( writeDelayMs );
+            things.add( em.create( "thing", new HashMap<String, Object>() {{
+                put( "name", thingName );
+                put( "updateCount", updateCount.getAndIncrement() );
+            }} ) );
+
         }
         em.refreshIndex();
 
-        CandidateResults crs = queryCollectionCp( "things", "thing", "select *");
+        CandidateResults crs = queryCollectionCp( "things", "thing", "select * order by updateCount
asc" );
         Assert.assertEquals( "Expect no stale candidates yet", numEntities, crs.size() );
 
         // update each one a bunch of times
         int count = 0;
 
-        List<Entity> maxVersions = new ArrayList<>(numEntities);
+        List<Entity> maxVersions = new ArrayList<>( numEntities );
+
+
+
 
         for ( Entity thing : things ) {
 
             Entity toUpdate = null;
 
-            for ( int j=0; j<numUpdates; j++) {
+            for ( int j = 0; j < numUpdates; j++ ) {
 
                 toUpdate = em.get( thing.getUuid() );
-                toUpdate.setProperty( "property"  + j, RandomStringUtils.randomAlphanumeric(10));
-                em.update(toUpdate);
+                //update the update count, so we'll order from the first entity created to
the last
+                toUpdate.setProperty( "updateCount", updateCount.getAndIncrement() );
+                em.update( toUpdate );
 
-                Thread.sleep( writeDelayMs );
 
                 count++;
                 if ( count % 100 == 0 ) {
-                    logger.info("Updated {} of {} times", count, numEntities * numUpdates);
+                    logger.info( "Updated {} of {} times", count, numEntities * numUpdates
);
                 }
             }
 
             maxVersions.add( toUpdate );
         }
+
         em.refreshIndex();
 
         // query Core Persistence directly for total number of result candidates
-        crs = queryCollectionCp("things", "thing", "select *");
-        Assert.assertEquals( "Expect stale candidates", numEntities * (numUpdates + 1), crs.size());
+        crs = queryCollectionCp( "things", "thing", "select * order by updateCount asc" );
+        Assert.assertEquals( "Expect stale candidates", numEntities * ( numUpdates + 1 ),
crs.size() );
 
         // query EntityManager for results and page through them
         // should return numEntities becuase it filters out the stale entities
-        final int limit  = 8;
-        Query q = Query.fromQL("select *");
+        final int limit = 8;
+
+        //we order by updateCount asc, this forces old versions to appear first, otherwise,
we don't clean them up in
+        // our versions
+        Query q = Query.fromQL( "select * order by updateCount asc" );
         q.setLimit( limit );
         int thingCount = 0;
         String cursor = null;
 
         int index = 0;
 
+
         do {
-            Results results = em.searchCollection( em.getApplicationRef(), "things", q);
+            Results results = em.searchCollection( em.getApplicationRef(), "things", q );
             thingCount += results.size();
 
-            logger.debug("Retrieved total of {} entities", thingCount );
+            logger.debug( "Retrieved total of {} entities", thingCount );
 
             cursor = results.getCursor();
             if ( cursor != null && thingCount < numEntities ) {
                 assertEquals( limit, results.size() );
             }
 
-            for (int i = 0; i < results.size(); i ++, index++){
+            for ( int i = 0; i < results.size(); i++, index++ ) {
 
-                final Entity returned = results.getEntities().get( i);
+                final Entity returned = results.getEntities().get( i );
 
                 //last entities appear first
                 final Entity expected = maxVersions.get( index );
-                assertEquals("correct entity returned", expected, returned);
-
+                assertEquals( "correct entity returned", expected, returned );
             }
-
-        } while ( cursor != null );
+        }
+        while ( cursor != null );
 
         assertEquals( "Expect no stale candidates", numEntities, thingCount );
 
+
         em.refreshIndex();
 
-        // EntityManager should have kicked off a batch cleanup of those stale indexes
-        // wait a second for batch cleanup to complete
-        Thread.sleep(600);
 
         // query for total number of result candidates = numEntities
-        crs = queryCollectionCp("things", "thing", "select *");
-        Assert.assertEquals( "Expect stale candidates de-indexed", numEntities, crs.size());
+        crs = queryCollectionCp( "things", "thing", "select *" );
+        Assert.assertEquals( "Expect stale candidates de-indexed", numEntities, crs.size()
);
     }
 
 
-    /** 
+    /**
      * Go around EntityManager and get directly from Core Persistence.
      */
     private org.apache.usergrid.persistence.model.entity.Entity getCpEntity( EntityRef eref
) {
 
         EntityManager em = app.getEntityManager();
 
-        CollectionScope cs = new CollectionScopeImpl(
-            new SimpleId( em.getApplicationId(), TYPE_APPLICATION),
-            new SimpleId( em.getApplicationId(), TYPE_APPLICATION),
-            CpNamingUtils.getCollectionScopeNameFromEntityType( eref.getType() ));
+        CollectionScope cs = new CollectionScopeImpl( new SimpleId( em.getApplicationId(),
TYPE_APPLICATION ),
+                new SimpleId( em.getApplicationId(), TYPE_APPLICATION ),
+                CpNamingUtils.getCollectionScopeNameFromEntityType( eref.getType() ) );
 
-        EntityCollectionManagerFactory ecmf = 
-                CpSetup.getInjector().getInstance( EntityCollectionManagerFactory.class );
+        EntityCollectionManagerFactory ecmf = CpSetup.getInjector().getInstance( EntityCollectionManagerFactory.class
);
 
-        EntityCollectionManager ecm = ecmf.createCollectionManager(cs);
+        EntityCollectionManager ecm = ecmf.createCollectionManager( cs );
 
-        return ecm.load( new SimpleId( eref.getUuid(), eref.getType()))
-                .toBlocking().lastOrDefault(null);
-    } 
+        return ecm.load( new SimpleId( eref.getUuid(), eref.getType() ) ).toBlocking().lastOrDefault(
null );
+    }
 
 
-    /** 
-     * Go around EntityManager and execute query directly against Core Persistence.
-     * Results may include stale index entries.
+    /**
+     * Go around EntityManager and execute query directly against Core Persistence. Results
may include stale index
+     * entries.
      */
-    private CandidateResults queryCollectionCp(final  String collName, final String type,
final String query ) {
+    private CandidateResults queryCollectionCp( final String collName, final String type,
final String query ) {
 
         EntityManager em = app.getEntityManager();
 
         EntityIndexFactory eif = CpSetup.getInjector().getInstance( EntityIndexFactory.class
);
 
-        ApplicationScope as = new ApplicationScopeImpl( 
-            new SimpleId( em.getApplicationId(), TYPE_APPLICATION));
+        ApplicationScope as = new ApplicationScopeImpl( new SimpleId( em.getApplicationId(),
TYPE_APPLICATION ) );
         EntityIndex ei = eif.createEntityIndex( as );
 
-        IndexScope is = new IndexScopeImpl(
-            new SimpleId( em.getApplicationId(), TYPE_APPLICATION),
-            CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ) );
-        Query rcq = Query.fromQL(query);
-        rcq.setLimit(10000); // no paging
+        IndexScope is = new IndexScopeImpl( new SimpleId( em.getApplicationId(), TYPE_APPLICATION
),
+                CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ) );
+        Query rcq = Query.fromQL( query );
+        rcq.setLimit( 10000 ); // no paging
 
         return ei.search( is, SearchTypes.fromTypes( type ), rcq );
     }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75123da3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index e6920f5..93ec9e3 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -58,6 +58,7 @@ import org.apache.usergrid.persistence.model.field.UUIDField;
 import org.apache.usergrid.persistence.model.field.value.EntityObject;
 
 import com.google.common.base.Joiner;
+import com.sun.xml.internal.ws.developer.MemberSubmissionAddressing;
 
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
@@ -135,7 +136,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
 
         String indexId = createIndexDocId( entity, context );
 
-        log.debug( "Indexing entity id {} data {} ", indexId, entityAsMap );
+        log.debug( "Indexing entity documentId {} data {} ", indexId, entityAsMap );
 
         bulkRequest.add( client.prepareIndex( indexName, entityType, indexId ).setSource(
entityAsMap ) );
 
@@ -149,20 +150,25 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
     public EntityIndexBatch deindex( final IndexScope indexScope, final Id id, final UUID
version ) {
 
         IndexValidationUtils.validateIndexScope( indexScope );
+        ValidationUtils.verifyIdentity( id );
+        ValidationUtils.verifyVersion( version );
 
         final String context = createContextName( indexScope );
         final String entityType = id.getType();
 
+        String indexId = createIndexDocId( id, version, context );
+
 
         if ( log.isDebugEnabled() ) {
-            log.debug( "De-indexing entity {}:{} in scope\n   app {}\n   owner {}\n   " +
"name {} context{}, type {}",
+            log.debug( "De-indexing entity {}:{} in scope\n   app {}\n   owner {}\n   " +
"name {} context{}, type {},",
                     new Object[] {
                             id.getType(), id.getUuid(), applicationScope.getApplication(),
indexScope.getOwner(),
                             indexScope.getName(), context, entityType
                     } );
         }
 
-        String indexId = createIndexDocId( id, version, context );
+
+        log.debug( "De-indexing type {} with documentId '{}'" , entityType, indexId);
 
         bulkRequest.add( client.prepareDelete( indexName, entityType, indexId ).setRefresh(
refresh ) );
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75123da3/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
index 20dcbde..18c3494 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
@@ -57,8 +57,8 @@ public class EntityConnectionIndexImplTest extends BaseIT {
 
     private static final Logger log = LoggerFactory.getLogger( EntityConnectionIndexImplTest.class
);
 
-//    @ClassRule
-//    public static ElasticSearchResource es = new ElasticSearchResource();
+    //    @ClassRule
+    //    public static ElasticSearchResource es = new ElasticSearchResource();
 
 
     @Inject
@@ -101,7 +101,7 @@ public class EntityConnectionIndexImplTest extends BaseIT {
 
 
         // create a person who likes muffins
-        Id personId =   new SimpleId( UUIDGenerator.newTimeUUID(), "person" ) ;
+        Id personId = new SimpleId( UUIDGenerator.newTimeUUID(), "person" );
 
 
         assertNotNull( personId );
@@ -113,7 +113,8 @@ public class EntityConnectionIndexImplTest extends BaseIT {
         IndexScope searchScope = new IndexScopeImpl( personId, "likes" );
 
         //create another scope we index in, want to be sure these scopes are filtered
-        IndexScope otherIndexScope = new IndexScopeImpl( new SimpleId( UUIDGenerator.newTimeUUID(),
"animal" ), "likes" );
+        IndexScope otherIndexScope =
+                new IndexScopeImpl( new SimpleId( UUIDGenerator.newTimeUUID(), "animal" ),
"likes" );
 
         EntityIndex personLikesIndex = ecif.createEntityIndex( applicationScope );
 
@@ -164,42 +165,134 @@ public class EntityConnectionIndexImplTest extends BaseIT {
         assertContains( muffin.getId(), likes );
         assertContains( oj.getId(), likes );
 
-          //now lets search for all explicitly
-        likes = personLikesIndex.search( searchScope,
-                SearchTypes.allTypes(),
-                Query.fromQL( "select *" ) );
+        //now lets search for all explicitly
+        likes = personLikesIndex.search( searchScope, SearchTypes.allTypes(), Query.fromQL(
"select *" ) );
         assertEquals( 3, likes.size() );
         assertContains( egg.getId(), likes );
-        assertContains( muffin.getId(), likes);
+        assertContains( muffin.getId(), likes );
         assertContains( oj.getId(), likes );
 
 
-
         //now search all entity types with a query that returns a subset
         likes = personLikesIndex.search( searchScope,
                 SearchTypes.fromTypes( muffin.getId().getType(), egg.getId().getType(), oj.getId().getType()
),
                 Query.fromQL( "select * where stars = 5" ) );
         assertEquals( 2, likes.size() );
-        assertContains( egg.getId(), likes);
+        assertContains( egg.getId(), likes );
         assertContains( muffin.getId(), likes );
 
 
-
         //now search with no types, we should get only the results that match
-        likes = personLikesIndex.search( searchScope, SearchTypes.allTypes(), Query.fromQL(
"select * where stars = 5" ) );
+        likes = personLikesIndex
+                .search( searchScope, SearchTypes.allTypes(), Query.fromQL( "select * where
stars = 5" ) );
         assertEquals( 2, likes.size() );
-        assertContains( egg.getId(), likes);
+        assertContains( egg.getId(), likes );
+        assertContains( muffin.getId(), likes );
+    }
+
+
+    @Test
+    public void testDelete() throws IOException {
+
+        Id appId = new SimpleId( "application" );
+        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+        // create a muffin
+        Entity muffin = new Entity( new SimpleId( UUIDGenerator.newTimeUUID(), "muffin" )
);
+
+        muffin = EntityIndexMapUtils.fromMap( muffin, new HashMap<String, Object>()
{{
+            put( "size", "Large" );
+            put( "flavor", "Blueberry" );
+            put( "stars", 5 );
+        }} );
+        EntityUtils.setVersion( muffin, UUIDGenerator.newTimeUUID() );
+
+        Entity egg = new Entity( new SimpleId( UUIDGenerator.newTimeUUID(), "egg" ) );
+
+        egg = EntityIndexMapUtils.fromMap( egg, new HashMap<String, Object>() {{
+            put( "size", "Large" );
+            put( "type", "scramble" );
+            put( "stars", 5 );
+        }} );
+        EntityUtils.setVersion( egg, UUIDGenerator.newTimeUUID() );
+
+        Entity oj = new Entity( new SimpleId( UUIDGenerator.newTimeUUID(), "juice" ) );
+
+        oj = EntityIndexMapUtils.fromMap( oj, new HashMap<String, Object>() {{
+            put( "size", "Large" );
+            put( "type", "pulpy" );
+            put( "stars", 3 );
+        }} );
+        EntityUtils.setVersion( oj, UUIDGenerator.newTimeUUID() );
+
+
+        // create a person who likes muffins
+        Id personId = new SimpleId( UUIDGenerator.newTimeUUID(), "person" );
+
+
+        assertNotNull( personId );
+        assertNotNull( personId.getType() );
+        assertNotNull( personId.getUuid() );
+
+        // index connection of "person Dave likes Large Blueberry muffin"
+
+        IndexScope searchScope = new IndexScopeImpl( personId, "likes" );
+
+        //create another scope we index in, want to be sure these scopes are filtered
+        IndexScope otherIndexScope =
+                new IndexScopeImpl( new SimpleId( UUIDGenerator.newTimeUUID(), "animal" ),
"likes" );
+
+        EntityIndex personLikesIndex = ecif.createEntityIndex( applicationScope );
+
+        EntityIndexBatch batch = personLikesIndex.createBatch();
+
+        //add to both scopes
+
+        //add a muffin
+        batch.index( searchScope, muffin );
+        batch.index( otherIndexScope, muffin );
+
+        //add the eggs
+        batch.index( searchScope, egg );
+        batch.index( otherIndexScope, egg );
+
+        //add the oj
+        batch.index( searchScope, oj );
+        batch.index( otherIndexScope, oj );
+
+        batch.executeAndRefresh();
+        personLikesIndex.refresh();
+
+        // now, let's search for muffins
+        CandidateResults likes = personLikesIndex.search( searchScope,
+                SearchTypes.fromTypes( muffin.getId().getType(), egg.getId().getType(), oj.getId().getType()
),
+                Query.fromQL( "select *" ) );
+        assertEquals( 3, likes.size() );
+        assertContains( egg.getId(), likes );
         assertContains( muffin.getId(), likes );
+        assertContains( oj.getId(), likes );
+
+
+        //now delete them
+        batch.deindex( searchScope, egg );
+        batch.deindex( searchScope, muffin );
+        batch.deindex( searchScope, oj );
+        batch.executeAndRefresh();
+
+        likes = personLikesIndex.search( searchScope,
+                SearchTypes.fromTypes( muffin.getId().getType(), egg.getId().getType(), oj.getId().getType()
),
+                Query.fromQL( "select *" ) );
+        assertEquals( 0, likes.size() );
     }
 
 
-    private void assertContains(final Id id, final CandidateResults results ){
-        for( CandidateResult result: results){
-            if(result.getId().equals(id)){
+    private void assertContains( final Id id, final CandidateResults results ) {
+        for ( CandidateResult result : results ) {
+            if ( result.getId().equals( id ) ) {
                 return;
             }
         }
 
-        fail(String.format("Could not find id %s in candidate results", id));
+        fail( String.format( "Could not find id %s in candidate results", id ) );
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75123da3/stack/rest/src/test/java/org/apache/usergrid/rest/RestITSuite.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/RestITSuite.java b/stack/rest/src/test/java/org/apache/usergrid/rest/RestITSuite.java
index 788a4b3..3c737a6 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/RestITSuite.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/RestITSuite.java
@@ -57,7 +57,7 @@ import org.apache.usergrid.rest.management.organizations.AdminEmailEncodingIT;
 public class RestITSuite {
 
     @ClassRule
-    public static ElasticSearchResource elasticSearchResource = new ElasticSearchResource().startEs();
+    public static ElasticSearchResource elasticSearchResource = new ElasticSearchResource();
 
     @ClassRule
     public static CassandraResource cassandraResource = CassandraResource.newWithAvailablePorts();

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75123da3/stack/services/src/test/java/org/apache/usergrid/ServiceITSuite.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/ServiceITSuite.java b/stack/services/src/test/java/org/apache/usergrid/ServiceITSuite.java
index d4e9751..5edc4579 100644
--- a/stack/services/src/test/java/org/apache/usergrid/ServiceITSuite.java
+++ b/stack/services/src/test/java/org/apache/usergrid/ServiceITSuite.java
@@ -60,5 +60,5 @@ public class ServiceITSuite {
     public static CassandraResource cassandraResource = CassandraResource.newWithAvailablePorts();
     
     @ClassRule
-    public static ElasticSearchResource elasticSearchResource = new ElasticSearchResource().startEs();
+    public static ElasticSearchResource elasticSearchResource = new ElasticSearchResource();
 }


Mime
View raw message