usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From toddn...@apache.org
Subject [07/13] git commit: Improvements to test, which now indexes data, reads data, deletes application indexes, rebuilds index and verifies that data is readable again.
Date Thu, 02 Oct 2014 05:44:17 GMT
Improvements to test, which now indexes data, reads data, deletes application indexes, rebuilds
index and verifies that data is readable again.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/df8c6b58
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/df8c6b58
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/df8c6b58

Branch: refs/heads/esbatching
Commit: df8c6b585c03040e3ba2819ef76a2875eea8f982
Parents: 352d69e
Author: Dave Johnson <dmjohnson@apigee.com>
Authored: Wed Oct 1 14:08:11 2014 -0400
Committer: Dave Johnson <dmjohnson@apigee.com>
Committed: Wed Oct 1 14:08:11 2014 -0400

----------------------------------------------------------------------
 .../corepersistence/CpEntityManager.java        |  40 +++++-
 .../corepersistence/CpEntityManagerFactory.java |   5 +-
 .../corepersistence/CpRelationManager.java      |   4 +-
 .../PerformanceEntityRebuildIndexTest.java      | 124 ++++++++++++++++---
 .../index/impl/EsEntityIndexImpl.java           |  18 +++
 5 files changed, 167 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/df8c6b58/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 49be5c3..38453e1 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2907,12 +2907,27 @@ public class CpEntityManager implements EntityManager {
                             CollectionScope collScope = new CollectionScopeImpl(
                                 applicationScope.getApplication(),
                                 applicationScope.getApplication(),
-                                CpEntityManager.getCollectionScopeNameFromCollectionName(collName));
+                                CpEntityManager.getCollectionScopeNameFromEntityType(entity.getType()));
                             EntityCollectionManager collMgr = 
                                 managerCache.getEntityCollectionManager(collScope);
 
                             org.apache.usergrid.persistence.model.entity.Entity collEntity
= 
-                                collMgr.load( edge.getTargetNode()).toBlockingObservable().last();
+                                collMgr.load( edge.getSourceNode() ).toBlockingObservable().last();
+
+//                            if (collEntity == null) {
+//                                if (logger.isDebugEnabled()) {
+//                                    logger.error("FAILED to load entity {}:{} "
+//                                            + "from scope\n   app {}\n   owner {}\n   name
{}",
+//                                            new Object[]{
+//                                                edge.getSourceNode().getType(), 
+//                                                edge.getSourceNode().getUuid(),
+//                                                collScope.getApplication(),
+//                                                collScope.getOwner(),
+//                                                collScope.getName()
+//                                            });
+//                                }
+//                                return;
+//                            }
 
                             CollectionScope memberScope = new CollectionScopeImpl(
                                 applicationScope.getApplication(),
@@ -2924,6 +2939,21 @@ public class CpEntityManager implements EntityManager {
                             org.apache.usergrid.persistence.model.entity.Entity memberEntity
= 
                                 memberMgr.load( edge.getTargetNode()).toBlockingObservable().last();
 
+//                            if (memberEntity == null) {
+//                                if (logger.isDebugEnabled()) {
+//                                    logger.error("FAILED to load entity {}:{} "
+//                                            + "from scope\n   app {}\n   owner {}\n   name
{}",
+//                                            new Object[]{
+//                                                edge.getTargetNode().getType(), 
+//                                                edge.getTargetNode().getUuid(),
+//                                                memberScope.getApplication(),
+//                                                memberScope.getOwner(),
+//                                                memberScope.getName()
+//                                            });
+//                                }
+//                                return;
+//                            }
+
                             indexEntityIntoCollections( collEntity, memberEntity, collName,
true );
 
                             EntityRef ref = new SimpleEntityRef( 
@@ -2948,7 +2978,7 @@ public class CpEntityManager implements EntityManager {
                                 managerCache.getEntityCollectionManager(sourceScope);
 
                             org.apache.usergrid.persistence.model.entity.Entity sourceEntity
= 
-                                sourceEcm.load( edge.getTargetNode()).toBlockingObservable().last();
+                                sourceEcm.load( fromEntityId ).toBlockingObservable().last();
 
                             CollectionScope targetScope = new CollectionScopeImpl(
                                 applicationScope.getApplication(),
@@ -2998,6 +3028,10 @@ public class CpEntityManager implements EntityManager {
                 .getCollection( memberEntity.getId().getType(), collName);
 
         if (connectBack && collection != null && collection.getLinkedCollection()
!= null) {
+
+            logger.debug("Linking back from entity in collection {} to collection {}", 
+                collection.getName(), collection.getLinkedCollection());
+
             indexEntityIntoCollections( 
                 memberEntity, collectionEntity, collection.getLinkedCollection(), false );
         }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/df8c6b58/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 00a808c..b3d5135 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -597,11 +597,13 @@ public class CpEntityManagerFactory implements EntityManagerFactory,
Application
     @Override
     public void rebuildApplicationIndexes( UUID appId, ProgressObserver po ) throws Exception
{
         
-        logger.info("Rebuilding index for application id {}", appId);
         EntityManager em = getEntityManager( appId );
         Application app = em.getApplication();
 
         ((CpEntityManager)em).reindex( po );
+        em.refreshIndex();
+
+        logger.info("\n\nRebuilt index for application {} id {}\n", app.getName(), appId
);
     }
 
 
@@ -616,6 +618,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
 
     @Override
     public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver po
) {
+        throw new UnsupportedOperationException( "Not supported yet." );
     }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/df8c6b58/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 04ed701..be21e85 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -916,7 +916,7 @@ public class CpRelationManager implements RelationManager {
                 results.merge( newResults );
             }
 
-            if ( crs.isEmpty() ) { // no more results
+            if ( crs.isEmpty() || !crs.hasCursor() ) { // no results, no cursor, can't get
more
                 satisfied = true;
 
             } else if ( results.size() == query.getLimit() )  { // got what we need
@@ -933,7 +933,7 @@ public class CpRelationManager implements RelationManager {
                 logger.warn("Satisfy query limit {}, new limit {} query count {}", new Object[]
{
                     originalLimit, query.getLimit(), queryCount 
                 });
-            }
+            } 
         }
 
         return results;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/df8c6b58/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index 7af66d3..663de59 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@ -33,11 +33,24 @@ import org.apache.usergrid.CoreApplication;
 import com.codahale.metrics.Meter;
 import com.codahale.metrics.MetricRegistry;
 import com.codahale.metrics.Slf4jReporter;
+import com.google.inject.Injector;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 import org.apache.usergrid.cassandra.Concurrent;
+import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
+import org.apache.usergrid.corepersistence.CpSetup;
+import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
+import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
+import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
 
@@ -52,7 +65,7 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
 
     private static final long RUNTIME = TimeUnit.MINUTES.toMillis( 1 );
 
-    private static final long writeDelayMs = 9;
+    private static final long writeDelayMs = 15;
     //private static final long readDelayMs = 7;
 
     @Rule
@@ -81,12 +94,15 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT
{
 
 
     @Test
-    public void rebuildIndex() {
+    public void rebuildIndex() throws Exception {
 
         logger.info("Started rebuildIndex()");
 
         final EntityManager em = app.getEntityManager();
-        final long stopTime = System.currentTimeMillis() + RUNTIME;
+
+        // ----------------- create a bunch of entities
+
+        final long stopTime = System.currentTimeMillis() + 300; // + RUNTIME;
         final Map<String, Object> entityMap = new HashMap<>();
 
         entityMap.put( "key1", 1000 );
@@ -94,29 +110,48 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT
{
         entityMap.put( "key3", "Some value" );
 
         List<EntityRef> entityRefs = new ArrayList<EntityRef>();
-
-        int i = 0;
+        int entityCount = 0;
         while ( System.currentTimeMillis() < stopTime ) {
-
-            entityMap.put( "key", i );
+            entityMap.put("key", entityCount );
             final Entity created;
             try {
                 created = em.create("testType", entityMap );
             } catch (Exception ex) {
                 throw new RuntimeException("Error creating entity", ex);
             }
-
             entityRefs.add( new SimpleEntityRef( created.getType(), created.getUuid() ) );
-
-            if ( i % 100 == 0 ) {
-                logger.info("Created {} entities", i );
+            if ( entityCount % 100 == 0 ) {
+                logger.info("Created {} entities", entityCount );
             }
-            i++;
-
+            entityCount++;
             try { Thread.sleep( writeDelayMs ); } catch (InterruptedException ignored ) {}
         }
-        logger.info("Created {} entities", i);
+        logger.info("Created {} entities", entityCount);
+        em.refreshIndex();
+
+        // ----------------- test that we can read them, should work fine 
+
+        logger.debug("Read the data");
+        readData("testTypes", entityCount );
+
+        // ----------------- delete the system and application indexes
 
+        logger.debug("Deleting app index and system app index");
+        deleteIndex( CpEntityManagerFactory.SYSTEM_APP_ID );
+        deleteIndex( em.getApplicationId() );
+
+        // ----------------- test that we can read them, should fail
+
+        logger.debug("Reading data, should fail this time ");
+        try {
+            readData( "testTypes", entityCount );
+            fail("should have failed to read data");
+
+        } catch (Exception expected) {}
+
+        // ----------------- rebuild index
+
+        logger.debug("Preparing to rebuild all indexes");;
 
         final String meterName = this.getClass().getSimpleName() + ".rebuildIndex";
         final Meter meter = registry.meter( meterName );
@@ -125,12 +160,9 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT
{
             int counter = 0;
             @Override
             public void onProgress( EntityRef s, EntityRef t, String etype ) {
-
                 meter.mark();
-
                 logger.debug("Indexing from {}:{} to {}:{} edgeType {}", new Object[] {
                     s.getType(), s.getUuid(), t.getType(), t.getUuid(), etype });
-
                 if ( !logger.isDebugEnabled() && counter % 100 == 0 ) {
                     logger.info("Reindexed {} entities", counter );
                 }
@@ -142,12 +174,68 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT
{
             setup.getEmf().rebuildAllIndexes( po );
 
             registry.remove( meterName );
-            logger.info("Finished rebuildIndex()");
+            logger.info("Rebuilt index");
 
         } catch (Exception ex) {
             logger.error("Error rebuilding index", ex);
             fail();
         }
 
+        // ----------------- test that we can read them
+        
+        readData( "testTypes", entityCount );
+    }
+
+    /** 
+     * Delete index for all applications, just need the one to get started.
+     */
+    private void deleteIndex( UUID appUuid ) {
+
+        Injector injector = CpSetup.getInjector();
+        EntityIndexFactory eif = injector.getInstance( EntityIndexFactory.class );
+
+        Id appId = new SimpleId( appUuid, "application");
+        IndexScope is = new IndexScopeImpl( appId, appId, "application");
+        EntityIndex ei = eif.createEntityIndex(is);
+        EsEntityIndexImpl eeii = (EsEntityIndexImpl)ei;
+
+        eeii.deleteIndex();
+    }
+
+    private int readData( String collectionName ) throws Exception {
+        return readData( collectionName, -1 );
+    }
+
+    private int readData( String collectionName, int expected ) throws Exception {
+
+        EntityManager em = app.getEntityManager();
+
+        Query q = Query.fromQL("select * where key1=1000");
+        q.setLimit(40);
+        Results results = em.searchCollection( em.getApplicationRef(), collectionName, q
);
+
+        int count = 0;
+        while ( true ) {
+
+            for ( Entity e : results.getEntities() ) {
+                assertEquals( 2000, e.getProperty("key2"));
+                //if ( count % 100 == 0 ) {
+                    logger.info( "read {} entities", count++);
+                //}
+            }
+
+            if ( results.hasCursor() ) {
+                logger.info( "Counted {} : query again with cursor", count);
+                q.setCursor( results.getCursor() );
+                results = em.searchCollection( em.getApplicationRef(), collectionName, q
);
+            } else {
+                break;
+            }
+        }
+
+        if ( expected != -1 && expected != count ) {
+            throw new RuntimeException("Did not get expected " + expected + " entities");
+        }
+        return count;
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/df8c6b58/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 8401e13..2eb8414 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -58,7 +58,10 @@ import org.apache.usergrid.persistence.model.field.SetField;
 import org.apache.usergrid.persistence.model.field.StringField;
 import org.apache.usergrid.persistence.model.field.UUIDField;
 import org.apache.usergrid.persistence.model.field.value.EntityObject;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
@@ -671,11 +674,13 @@ public class EsEntityIndexImpl implements EntityIndex {
     }
 
 
+    @Override
     public void refresh() {
         client.admin().indices().prepareRefresh( indexName ).execute().actionGet();
         log.debug("Refreshed index: " + indexName);
     }
 
+
     @Override
     public CandidateResults getEntityVersions(Id id) {
         Query query = new Query();
@@ -684,4 +689,17 @@ public class EsEntityIndexImpl implements EntityIndex {
         return results;
     }
 
+    /**
+     * For testing only.
+     */
+    public void deleteIndex() {
+        AdminClient adminClient = client.admin();
+        DeleteIndexResponse response = adminClient.indices().prepareDelete( indexName ).get();
+        if ( response.isAcknowledged() ) {
+            log.info("Deleted index: " + indexName );
+        } else {
+            log.info("Failed to delete index " + indexName );
+        }
+    }
+
 }


Mime
View raw message