Return-Path: X-Original-To: apmail-usergrid-commits-archive@minotaur.apache.org Delivered-To: apmail-usergrid-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 73594172BA for ; Mon, 20 Oct 2014 13:46:32 +0000 (UTC) Received: (qmail 10971 invoked by uid 500); 20 Oct 2014 13:46:30 -0000 Delivered-To: apmail-usergrid-commits-archive@usergrid.apache.org Received: (qmail 10896 invoked by uid 500); 20 Oct 2014 13:46:30 -0000 Mailing-List: contact commits-help@usergrid.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@usergrid.incubator.apache.org Delivered-To: mailing list commits@usergrid.incubator.apache.org Received: (qmail 10339 invoked by uid 99); 20 Oct 2014 13:46:29 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 20 Oct 2014 13:46:29 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 84A1A9B1618; Mon, 20 Oct 2014 13:46:29 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: snoopdave@apache.org To: commits@usergrid.apache.org Date: Mon, 20 Oct 2014 13:46:49 -0000 Message-Id: In-Reply-To: <3886f80d5df9455c8ecb83dfd658e8af@git.apache.org> References: <3886f80d5df9455c8ecb83dfd658e8af@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [21/50] [abbrv] git commit: Added verification to init to catch NPE issues Added verification to init to catch NPE issues Updated index creation to write a document and delete to verify correct functionality Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/722c43b7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/722c43b7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/722c43b7 Branch: refs/heads/two-dot-o-events Commit: 722c43b731a80c810916a09af8e50c19ac7c6891 Parents: 83200bf Author: Todd Nine Authored: Thu Oct 16 20:53:59 2014 -0600 Committer: Todd Nine Committed: Thu Oct 16 20:53:59 2014 -0600 ---------------------------------------------------------------------- .../corepersistence/CpEntityManager.java | 20 +-- .../usergrid/persistence/EntityManager.java | 2 +- .../index/impl/EsEntityIndexImpl.java | 144 ++++++++++++++----- 3 files changed, 121 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/722c43b7/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java index 2498dda..751b9e2 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java @@ -16,6 +16,7 @@ package org.apache.usergrid.corepersistence; +import com.google.common.base.Preconditions; import com.netflix.hystrix.exception.HystrixRuntimeException; import com.yammer.metrics.annotation.Metered; import static java.lang.String.CASE_INSENSITIVE_ORDER; @@ -191,6 +192,9 @@ public class CpEntityManager implements EntityManager { @Override public void init( EntityManagerFactory emf, UUID applicationId ) { + Preconditions.checkNotNull(emf, "emf must not be null"); + Preconditions.checkNotNull( applicationId, "applicationId must not be null" ); + this.emf = ( CpEntityManagerFactory ) emf; this.managerCache = this.emf.getManagerCache(); this.applicationId = applicationId; @@ -202,9 +206,6 @@ public class CpEntityManager implements EntityManager { // set to false for now this.skipAggregateCounters = false; - - - applicationScope = this.emf.getApplicationScope( applicationId ); } @@ -643,11 +644,11 @@ public class CpEntityManager implements EntityManager { return getRelationManager( entityRef ).searchCollection( collectionName, query ); } - - @Override - public void setApplicationId( UUID applicationId ) { - this.applicationId = applicationId; - } +// +// @Override +// public void setApplicationId( UUID applicationId ) { +// this.applicationId = applicationId; +// } @Override @@ -2884,7 +2885,8 @@ public class CpEntityManager implements EntityManager { } catch(WriteOptimisticVerifyException wo ){ //swallow this, it just means this was already updated, which accomplishes our task. Just ignore. - logger.warn( "Someone beat us to updating entity {} in collection {}. Ignoring.", entity.getName(), collName ); + logger.warn( "Someone beat us to updating entity {} in collection {}. Ignoring.", entity.getName(), + collName ); } catch (Exception ex) { logger.error("Error repersisting entity", ex); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/722c43b7/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java index 3684d7e..cd92729 100644 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java @@ -41,7 +41,7 @@ import org.apache.usergrid.persistence.index.query.Query.Level; */ public interface EntityManager { - public void setApplicationId( UUID applicationId ); +// public void setApplicationId( UUID applicationId ); public GeoIndexManager getGeoIndexManager(); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/722c43b7/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java index b562d8a..7584386 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java @@ -26,7 +26,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; -import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; @@ -36,7 +35,9 @@ import org.elasticsearch.client.Client; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.FilterBuilder; +import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.indices.IndexAlreadyExistsException; import org.elasticsearch.indices.IndexMissingException; import org.elasticsearch.search.SearchHit; @@ -58,7 +59,9 @@ import org.apache.usergrid.persistence.index.query.CandidateResults; import org.apache.usergrid.persistence.index.query.Query; import org.apache.usergrid.persistence.model.entity.Id; import org.apache.usergrid.persistence.model.entity.SimpleId; +import org.apache.usergrid.persistence.model.util.UUIDGenerator; +import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import com.google.inject.assistedinject.Assisted; @@ -88,11 +91,18 @@ public class EsEntityIndexImpl implements EntityIndex { private final IndexFig config; - //number of times to wait for the index to refresh propertly. Is an N+1, so 9 = 10 - private static final int MAX_WAITS = 9; + //number of times to wait for the index to refresh properly. + private static final int MAX_WAITS = 10; //number of milliseconds to try again before sleeping private static final int WAIT_TIME = 250; + private static final String VERIFY_TYPE = "verification"; + + private static final ImmutableMap DEFAULT_PAYLOAD = + ImmutableMap.of( "field", "test" ); + + private static final MatchAllQueryBuilder MATCH_ALL_QUERY_BUILDER = QueryBuilders.matchAllQuery(); + @Inject public EsEntityIndexImpl( @Assisted final ApplicationScope appScope, final IndexFig config, @@ -120,21 +130,16 @@ public class EsEntityIndexImpl implements EntityIndex { CreateIndexResponse cir = admin.indices().prepareCreate( indexName ).execute().actionGet(); log.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged() ); - RefreshResponse response; - - do { - response = admin.indices().prepareRefresh( indexName ).execute().actionGet(); - } - while ( response.getFailedShards() != 0 ); + //create the document, this ensures the index is ready /** - * Immediately refresh to ensure the entire cluster is ready to receive this write. Occasionally we see + * Immediately create a document and remove it to ensure the entire cluster is ready to receive documents + * . Occasionally we see * errors. See this post. - * http://elasticsearch-users.115913.n3.nabble - * .com/IndexMissingException-on-create-index-followed-by-refresh-td1832793.html + * http://elasticsearch-users.115913.n3.nabble.com/IndexMissingException-on-create-index-followed-by-refresh-td1832793.html * */ - refresh(); + testNewIndex(); } catch ( IndexAlreadyExistsException expected ) { // this is expected to happen if index already exists, it's a no-op and swallow @@ -146,6 +151,44 @@ public class EsEntityIndexImpl implements EntityIndex { /** + * Tests writing a document to a new index to ensure it's working correctly. Comes from email + * + * http://elasticsearch-users.115913.n3.nabble + * .com/IndexMissingException-on-create-index-followed-by-refresh-td1832793.html + */ + + private void testNewIndex() { + + + log.info( "Refreshing Created new Index Name [{}]", indexName ); + + final RetryOperation retryOperation = new RetryOperation() { + @Override + public boolean doOp() { + final String tempId = UUIDGenerator.newTimeUUID().toString(); + + + client.prepareIndex( indexName, VERIFY_TYPE, tempId ).setSource( DEFAULT_PAYLOAD ).get(); + + log.info( "Successfully created new document with docId {} in index {} and type {}", tempId, indexName, + VERIFY_TYPE ); + + //delete all types, this way if we miss one it will get cleaned up + + client.prepareDeleteByQuery( indexName ).setTypes( VERIFY_TYPE ).setQuery( MATCH_ALL_QUERY_BUILDER ) + .get(); + + log.info( "Successfully deleted all documents in index {} and type {}", indexName, VERIFY_TYPE ); + + return true; + } + }; + + doInRetry( retryOperation ); + } + + + /** * Setup ElasticSearch type mappings as a template that applies to all new indexes. Applies to all indexes that * start with our prefix. */ @@ -277,30 +320,22 @@ public class EsEntityIndexImpl implements EntityIndex { log.info( "Refreshing Created new Index Name [{}]", indexName ); - //now try to refresh, to ensure that it's recognized by everyone. Occasionally we can get a success - //before we can write. - for ( int i = 0; i < MAX_WAITS; i++ ) { - try { - client.admin().indices().prepareRefresh( indexName ).execute().actionGet(); - log.debug( "Refreshed index: " + indexName ); - return; - } - catch ( IndexMissingException e ) { - log.error( "Unable to refresh index after create. Waiting before sleeping.", e ); - } - - try { - Thread.sleep( WAIT_TIME ); - } - catch ( InterruptedException e ) { - //swallow it + final RetryOperation retryOperation = new RetryOperation() { + @Override + public boolean doOp() { + try { + client.admin().indices().prepareRefresh( indexName ).execute().actionGet(); + log.debug( "Refreshed index: " + indexName ); + return true; + } + catch ( IndexMissingException e ) { + log.error( "Unable to refresh index after create. Waiting before sleeping.", e ); + throw e; + } } - } + }; - /** - * Try the refresh one last time if we get here - */ - client.admin().indices().prepareRefresh( indexName ).execute().actionGet(); + doInRetry( retryOperation ); log.debug( "Refreshed index: " + indexName ); } @@ -328,4 +363,43 @@ public class EsEntityIndexImpl implements EntityIndex { log.info( "Failed to delete index " + indexName ); } } + + + /** + * Do the retry operation + * @param operation + */ + private void doInRetry( final RetryOperation operation ) { + for ( int i = 0; i < MAX_WAITS; i++ ) { + + try { + if(operation.doOp()){ + return; + } + } + catch ( Exception e ) { + log.error( "Unable to execute operation, retrying", e ); + } + + + try { + Thread.sleep( WAIT_TIME ); + } + catch ( InterruptedException e ) { + //swallow it + } + } + } + + + /** + * Interface for operations + */ + private static interface RetryOperation { + + /** + * Return true if done, false if there should be a retry + */ + public boolean doOp(); + } }