usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From snoopd...@apache.org
Subject [21/50] [abbrv] git commit: Added verification to init to catch NPE issues
Date Mon, 20 Oct 2014 13:46:49 GMT
Added verification to init to catch NPE issues

Updated index creation to write a document and delete to verify correct functionality


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/722c43b7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/722c43b7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/722c43b7

Branch: refs/heads/two-dot-o-events
Commit: 722c43b731a80c810916a09af8e50c19ac7c6891
Parents: 83200bf
Author: Todd Nine <toddnine@apache.org>
Authored: Thu Oct 16 20:53:59 2014 -0600
Committer: Todd Nine <toddnine@apache.org>
Committed: Thu Oct 16 20:53:59 2014 -0600

----------------------------------------------------------------------
 .../corepersistence/CpEntityManager.java        |  20 +--
 .../usergrid/persistence/EntityManager.java     |   2 +-
 .../index/impl/EsEntityIndexImpl.java           | 144 ++++++++++++++-----
 3 files changed, 121 insertions(+), 45 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/722c43b7/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 2498dda..751b9e2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -16,6 +16,7 @@
 package org.apache.usergrid.corepersistence;
 
 
+import com.google.common.base.Preconditions;
 import com.netflix.hystrix.exception.HystrixRuntimeException;
 import com.yammer.metrics.annotation.Metered;
 import static java.lang.String.CASE_INSENSITIVE_ORDER;
@@ -191,6 +192,9 @@ public class CpEntityManager implements EntityManager {
     @Override
     public void init( EntityManagerFactory emf, UUID applicationId ) {
 
+        Preconditions.checkNotNull(emf, "emf must not be null");
+        Preconditions.checkNotNull( applicationId, "applicationId must not be null"  );
+
         this.emf = ( CpEntityManagerFactory ) emf;
         this.managerCache = this.emf.getManagerCache();
         this.applicationId = applicationId;
@@ -202,9 +206,6 @@ public class CpEntityManager implements EntityManager {
 
         // set to false for now
         this.skipAggregateCounters = false;
-
-
-        applicationScope = this.emf.getApplicationScope( applicationId );
     }
 
 
@@ -643,11 +644,11 @@ public class CpEntityManager implements EntityManager {
         return getRelationManager( entityRef ).searchCollection( collectionName, query );
     }
 
-
-    @Override
-    public void setApplicationId( UUID applicationId ) {
-        this.applicationId = applicationId;
-    }
+//
+//    @Override
+//    public void setApplicationId( UUID applicationId ) {
+//        this.applicationId = applicationId;
+//    }
 
 
     @Override
@@ -2884,7 +2885,8 @@ public class CpEntityManager implements EntityManager {
                 }
                 catch(WriteOptimisticVerifyException wo ){
                     //swallow this, it just means this was already updated, which accomplishes
our task.  Just ignore.
-                    logger.warn( "Someone beat us to updating entity {} in collection {}.
 Ignoring.", entity.getName(), collName );
+                    logger.warn( "Someone beat us to updating entity {} in collection {}.
 Ignoring.", entity.getName(),
+                            collName );
                 }
                 catch (Exception ex) {
                     logger.error("Error repersisting entity", ex);

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/722c43b7/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index 3684d7e..cd92729 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -41,7 +41,7 @@ import org.apache.usergrid.persistence.index.query.Query.Level;
  */
 public interface EntityManager {
 
-    public void setApplicationId( UUID applicationId );
+//    public void setApplicationId( UUID applicationId );
 
     public GeoIndexManager getGeoIndexManager();
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/722c43b7/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index b562d8a..7584386 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -26,7 +26,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
-import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
@@ -36,7 +35,9 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.query.FilterBuilder;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.indices.IndexMissingException;
 import org.elasticsearch.search.SearchHit;
@@ -58,7 +59,9 @@ import org.apache.usergrid.persistence.index.query.CandidateResults;
 import org.apache.usergrid.persistence.index.query.Query;
 import org.apache.usergrid.persistence.model.entity.Id;
 import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
 
+import com.google.common.collect.ImmutableMap;
 import com.google.inject.Inject;
 import com.google.inject.assistedinject.Assisted;
 
@@ -88,11 +91,18 @@ public class EsEntityIndexImpl implements EntityIndex {
 
     private final IndexFig config;
 
-    //number of times to wait for the index to refresh propertly. Is an N+1, so 9 = 10
-    private static final int MAX_WAITS = 9;
+    //number of times to wait for the index to refresh properly.
+    private static final int MAX_WAITS = 10;
     //number of milliseconds to try again before sleeping
     private static final int WAIT_TIME = 250;
 
+    private static final String VERIFY_TYPE = "verification";
+
+    private static final ImmutableMap<String, Object> DEFAULT_PAYLOAD =
+            ImmutableMap.<String, Object>of( "field", "test" );
+
+    private static final MatchAllQueryBuilder MATCH_ALL_QUERY_BUILDER = QueryBuilders.matchAllQuery();
+
 
     @Inject
     public EsEntityIndexImpl( @Assisted final ApplicationScope appScope, final IndexFig config,
@@ -120,21 +130,16 @@ public class EsEntityIndexImpl implements EntityIndex {
             CreateIndexResponse cir = admin.indices().prepareCreate( indexName ).execute().actionGet();
             log.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged()
);
 
-            RefreshResponse response;
-
-            do {
-                response = admin.indices().prepareRefresh( indexName ).execute().actionGet();
-            }
-            while ( response.getFailedShards() != 0 );
 
+            //create the document, this ensures the index is ready
             /**
-             * Immediately refresh to ensure the entire cluster is ready to receive this
write.  Occasionally we see
+             * Immediately create a document and remove it to ensure the entire cluster is
ready to receive documents
+             * .  Occasionally we see
              * errors.  See this post.
-             * http://elasticsearch-users.115913.n3.nabble
-             * .com/IndexMissingException-on-create-index-followed-by-refresh-td1832793.html
+             * http://elasticsearch-users.115913.n3.nabble.com/IndexMissingException-on-create-index-followed-by-refresh-td1832793.html
              *
              */
-            refresh();
+            testNewIndex();
         }
         catch ( IndexAlreadyExistsException expected ) {
             // this is expected to happen if index already exists, it's a no-op and swallow
@@ -146,6 +151,44 @@ public class EsEntityIndexImpl implements EntityIndex {
 
 
     /**
+     * Tests writing a document to a new index to ensure it's working correctly.  Comes from
email
+     *
+     * http://elasticsearch-users.115913.n3.nabble
+     * .com/IndexMissingException-on-create-index-followed-by-refresh-td1832793.html
+     */
+
+    private void testNewIndex() {
+
+
+        log.info( "Refreshing Created new Index Name [{}]", indexName );
+
+        final RetryOperation retryOperation = new RetryOperation() {
+            @Override
+            public boolean doOp() {
+                final String tempId = UUIDGenerator.newTimeUUID().toString();
+
+
+                client.prepareIndex( indexName, VERIFY_TYPE, tempId ).setSource( DEFAULT_PAYLOAD
).get();
+
+                log.info( "Successfully created new document with docId {} in index {} and
type {}", tempId, indexName,
+                        VERIFY_TYPE );
+
+                //delete all types, this way if we miss one it will get cleaned up
+
+                client.prepareDeleteByQuery( indexName ).setTypes( VERIFY_TYPE ).setQuery(
MATCH_ALL_QUERY_BUILDER )
+                      .get();
+
+                log.info( "Successfully deleted all documents in index {} and type {}", indexName,
VERIFY_TYPE );
+
+                return true;
+            }
+        };
+
+        doInRetry( retryOperation );
+    }
+
+
+    /**
      * Setup ElasticSearch type mappings as a template that applies to all new indexes. Applies
to all indexes that
      * start with our prefix.
      */
@@ -277,30 +320,22 @@ public class EsEntityIndexImpl implements EntityIndex {
 
         log.info( "Refreshing Created new Index Name [{}]", indexName );
 
-        //now try to refresh, to ensure that it's recognized by everyone.  Occasionally we
can get a success
-        //before we can write.
-        for ( int i = 0; i < MAX_WAITS; i++ ) {
-            try {
-                client.admin().indices().prepareRefresh( indexName ).execute().actionGet();
-                log.debug( "Refreshed index: " + indexName );
-                return;
-            }
-            catch ( IndexMissingException e ) {
-                log.error( "Unable to refresh index after create. Waiting before sleeping.",
e );
-            }
-
-            try {
-                Thread.sleep( WAIT_TIME );
-            }
-            catch ( InterruptedException e ) {
-                //swallow it
+        final RetryOperation retryOperation = new RetryOperation() {
+            @Override
+            public boolean doOp() {
+                try {
+                    client.admin().indices().prepareRefresh( indexName ).execute().actionGet();
+                    log.debug( "Refreshed index: " + indexName );
+                    return true;
+                }
+                catch ( IndexMissingException e ) {
+                    log.error( "Unable to refresh index after create. Waiting before sleeping.",
e );
+                    throw e;
+                }
             }
-        }
+        };
 
-        /**
-         * Try the refresh one last time if we get here
-         */
-        client.admin().indices().prepareRefresh( indexName ).execute().actionGet();
+        doInRetry( retryOperation );
 
         log.debug( "Refreshed index: " + indexName );
     }
@@ -328,4 +363,43 @@ public class EsEntityIndexImpl implements EntityIndex {
             log.info( "Failed to delete index " + indexName );
         }
     }
+
+
+    /**
+     * Do the retry operation
+     * @param operation
+     */
+    private void doInRetry( final RetryOperation operation ) {
+        for ( int i = 0; i < MAX_WAITS; i++ ) {
+
+            try {
+                if(operation.doOp()){
+                    return;
+                }
+            }
+            catch ( Exception e ) {
+                log.error( "Unable to execute operation, retrying", e );
+            }
+
+
+            try {
+                Thread.sleep( WAIT_TIME );
+            }
+            catch ( InterruptedException e ) {
+                //swallow it
+            }
+        }
+    }
+
+
+    /**
+     * Interface for operations
+     */
+    private static interface RetryOperation {
+
+        /**
+         * Return true if done, false if there should be a retry
+         */
+        public boolean doOp();
+    }
 }


Mime
View raw message