usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sfeld...@apache.org
Subject [16/43] git commit: A optimization: set our ElasticSearch field mappings once at the cluster level, so we do not have to set them for every new type that is created.
Date Fri, 10 Oct 2014 15:50:23 GMT
A optimization: set our ElasticSearch field mappings once at the cluster level, so we do not
have to set them for every new type that is created.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c0edbe19
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c0edbe19
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c0edbe19

Branch: refs/heads/two-dot-o-events
Commit: c0edbe1914b1228dba6c7b06d2da7590def6da13
Parents: 81d4e0e
Author: Dave Johnson <dmjohnson@apigee.com>
Authored: Thu Oct 9 14:29:48 2014 -0400
Committer: Dave Johnson <dmjohnson@apigee.com>
Committed: Thu Oct 9 14:29:48 2014 -0400

----------------------------------------------------------------------
 .../index/impl/EsEntityIndexBatchImpl.java      | 117 ++++++++-----------
 .../index/impl/EsEntityIndexImpl.java           | 117 ++++++-------------
 .../persistence/index/impl/IndexingUtils.java   |  66 +++++++++++
 3 files changed, 145 insertions(+), 155 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c0edbe19/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index b88be98..9e70a00 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -18,7 +18,6 @@ package org.apache.usergrid.persistence.index.impl;/*
  */
 
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -30,10 +29,7 @@ import java.util.UUID;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.client.AdminClient;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -62,6 +58,7 @@ import org.apache.usergrid.persistence.model.field.UUIDField;
 import org.apache.usergrid.persistence.model.field.value.EntityObject;
 
 import com.google.common.base.Joiner;
+import java.io.IOException;
 
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
@@ -72,7 +69,10 @@ import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PR
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createCollectionScopeTypeName;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexDocId;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexName;
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.indices.IndexAlreadyExistsException;
 
 
 public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@@ -98,8 +98,9 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
     private int count;
 
 
-    public EsEntityIndexBatchImpl( final ApplicationScope applicationScope, final Client
client, final IndexFig config,
-                                   final Set<String> knownTypes, final int autoFlushSize
) {
+    public EsEntityIndexBatchImpl( 
+            final ApplicationScope applicationScope, final Client client, final IndexFig
config,
+            final Set<String> knownTypes, final int autoFlushSize ) {
 
         this.applicationScope = applicationScope;
         this.client = client;
@@ -120,20 +121,23 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
         final String indexType = createCollectionScopeTypeName( indexScope );
 
         if ( log.isDebugEnabled() ) {
-            log.debug( "Indexing entity {}:{} in scope\n   app {}\n   owner {}\n   name {}\n
  type {}", new Object[] {
-                    entity.getId().getType(), entity.getId().getUuid(), applicationScope.getApplication(),
-                    indexScope.getOwner(), indexScope.getName(), indexType
-            } );
+            log.debug( "Indexing entity {}:{} in scope\n   app {}\n   "
+                + "owner {}\n   name {}\n   type {}", new Object[] {
+                    entity.getId().getType(), 
+                    entity.getId().getUuid(), 
+                    applicationScope.getApplication(), 
+                    indexScope.getOwner(), 
+                    indexScope.getName(), indexType
+            });
         }
 
         ValidationUtils.verifyEntityWrite( entity );
 
-        initType( indexScope, indexType );
-
         Map<String, Object> entityAsMap = entityToMap( entity );
 
         // need prefix here becuase we index UUIDs as strings
-        entityAsMap.put( STRING_PREFIX + ENTITYID_FIELDNAME, entity.getId().getUuid().toString().toLowerCase()
);
+        entityAsMap.put( STRING_PREFIX + ENTITYID_FIELDNAME, 
+                entity.getId().getUuid().toString().toLowerCase() );
 
         // let caller add these fields if needed
         // entityAsMap.put("created", entity.getId().getUuid().timestamp();
@@ -143,7 +147,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
 
         log.debug( "Indexing entity id {} data {} ", indexId, entityAsMap );
 
-        bulkRequest.add( client.prepareIndex( indexName, indexType, indexId ).setSource(
entityAsMap ) );
+        bulkRequest.add(client.prepareIndex( indexName, indexType, indexId).setSource(entityAsMap));
 
         maybeFlush();
 
@@ -159,16 +163,20 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
         final String indexType = createCollectionScopeTypeName( indexScope );
 
         if ( log.isDebugEnabled() ) {
-            log.debug( "De-indexing entity {}:{} in scope\n   app {}\n   owner {}\n   name
{} type {}", new Object[] {
-                    id.getType(), id.getUuid(), applicationScope.getApplication(), indexScope.getOwner(),
-                    indexScope.getName(), indexType
+            log.debug( "De-indexing entity {}:{} in scope\n   app {}\n   owner {}\n   "
+                + "name {} type {}", new Object[] {
+                    id.getType(), 
+                    id.getUuid(), 
+                    applicationScope.getApplication(), 
+                    indexScope.getOwner(),
+                    indexScope.getName(), 
+                    indexType
             } );
         }
 
         String indexId = createIndexDocId( id, version );
 
-        bulkRequest.add( client.prepareDelete( indexName, indexType, indexId ).setRefresh(
refresh ) );
-
+        bulkRequest.add( client.prepareDelete( indexName, indexType, indexId ).setRefresh(refresh));
 
         log.debug( "Deindexed Entity with index id " + indexId );
 
@@ -212,8 +220,8 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
 
         for ( BulkItemResponse response : responses ) {
             if ( response.isFailed() ) {
-                throw new RuntimeException(
-                        "Unable to index documents.  Errors are :" + response.getFailure().getMessage()
);
+                throw new RuntimeException("Unable to index documents.  Errors are :" 
+                        + response.getFailure().getMessage() );
             }
         }
 
@@ -238,48 +246,13 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
 
 
     /**
-     * Create ElasticSearch mapping for each type of Entity.
-     */
-    private void initType( final IndexScope indexScope, String typeName ) {
-
-        // no need for synchronization here, it's OK if we init attempt to init type multiple
times
-        if ( knownTypes.contains( typeName ) ) {
-            return;
-        }
-
-        AdminClient admin = client.admin();
-        try {
-            XContentBuilder mxcb = EsEntityIndexImpl.createDoubleStringIndexMapping( jsonBuilder(),
typeName );
-
-
-            //TODO Dave can this be collapsed into the build as well?
-            admin.indices().preparePutMapping( indexName ).setType( typeName ).setSource(
mxcb ).execute().actionGet();
-
-            admin.indices().prepareGetMappings( indexName ).addTypes( typeName ).execute().actionGet();
-
-            //            log.debug("Created new type mapping");
-            //            log.debug("   Scope application: " + indexScope.getApplication());
-            //            log.debug("   Scope owner: " + indexScope.getOwner());
-            //            log.debug("   Type name: " + typeName);
-
-            knownTypes.add( typeName );
-        }
-        catch ( IndexAlreadyExistsException ignored ) {
-            // expected
-        }
-        catch ( IOException ex ) {
-            throw new RuntimeException(
-                    "Exception initializing type " + typeName + " in app " + applicationScope.getApplication()
-                                                                                        
    .toString() );
-        }
-    }
-
-
-    /**
-     * Convert Entity to Map. Adding prefixes for types:
-     *
-     * su_ - String unanalyzed field sa_ - String analyzed field go_ - Location field nu_
- Number field bu_ - Boolean
-     * field
+     * Convert Entity to Map and Adding prefixes for types:
+     * <pre>
+     * su_ - String unanalyzed field 
+     * sa_ - String analyzed field 
+     * go_ - Location field nu_ - Number field 
+     * bu_ - Boolean field
+     * </pre>
      */
     private static Map entityToMap( EntityObject entity ) {
 
@@ -291,7 +264,8 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
 
             if ( f instanceof ListField ) {
                 List list = ( List ) field.getValue();
-                entityMap.put( field.getName().toLowerCase(), new ArrayList( processCollectionForMap(
list ) ) );
+                entityMap.put( field.getName().toLowerCase(), 
+                        new ArrayList( processCollectionForMap( list ) ) );
 
                 if ( !list.isEmpty() ) {
                     if ( list.get( 0 ) instanceof String ) {
@@ -304,18 +278,17 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
             }
             else if ( f instanceof ArrayField ) {
                 List list = ( List ) field.getValue();
-                entityMap.put( field.getName().toLowerCase(), new ArrayList( processCollectionForMap(
list ) ) );
+                entityMap.put( field.getName().toLowerCase(), 
+                        new ArrayList( processCollectionForMap( list ) ) );
             }
             else if ( f instanceof SetField ) {
                 Set set = ( Set ) field.getValue();
-                entityMap.put( field.getName().toLowerCase(), new ArrayList( processCollectionForMap(
set ) ) );
+                entityMap.put( field.getName().toLowerCase(), 
+                        new ArrayList( processCollectionForMap( set ) ) );
             }
             else if ( f instanceof EntityObjectField ) {
                 EntityObject eo = ( EntityObject ) field.getValue();
                 entityMap.put( field.getName().toLowerCase(), entityToMap( eo ) ); // recursion
-
-                // Add type information as field-name prefixes
-
             }
             else if ( f instanceof StringField ) {
 
@@ -334,7 +307,9 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
                 locMap.put( "lon", locField.getValue().getLongitude() );
                 entityMap.put( GEO_PREFIX + field.getName().toLowerCase(), locMap );
             }
-            else if ( f instanceof DoubleField || f instanceof FloatField || f instanceof
IntegerField
+            else if ( f instanceof DoubleField 
+                    || f instanceof FloatField 
+                    || f instanceof IntegerField
                     || f instanceof LongField ) {
 
                 entityMap.put( NUMBER_PREFIX + field.getName().toLowerCase(), field.getValue()
);

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c0edbe19/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 23717b1..8c07d6d 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -17,7 +17,6 @@
  */
 package org.apache.usergrid.persistence.index.impl;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
@@ -57,16 +56,15 @@ import org.apache.usergrid.persistence.model.entity.SimpleId;
 
 import com.google.inject.Inject;
 import com.google.inject.assistedinject.Assisted;
+import java.io.IOException;
 
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.DOC_ID_SEPARATOR_SPLITTER;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_FIELDNAME;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.GEO_PREFIX;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createCollectionScopeTypeName;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexName;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
+import org.elasticsearch.common.xcontent.XContentFactory;
 
 
 /**
@@ -91,20 +89,18 @@ public class EsEntityIndexImpl implements EntityIndex {
     private final IndexFig config;
 
     @Inject
-    public EsEntityIndexImpl(@Assisted final ApplicationScope applicationScope, 
+    public EsEntityIndexImpl(@Assisted final ApplicationScope appScope, 
             final IndexFig config, final EsProvider provider) {
 
-        ValidationUtils.validateApplicationScope(applicationScope);
+        ValidationUtils.validateApplicationScope(appScope);
 
         try {
-            this.applicationScope = applicationScope;
-
+            this.applicationScope = appScope;
             this.client = provider.getClient();
-
-            this.indexName = createIndexName(config.getIndexPrefix(), applicationScope);
+            this.config = config;
             this.cursorTimeout = config.getQueryCursorTimeout();
+            this.indexName = IndexingUtils.createIndexName(config.getIndexPrefix(), appScope);
 
-            this.config = config;
         } catch (Exception e) {
             log.error("Error setting up index", e);
             throw e;
@@ -112,18 +108,34 @@ public class EsEntityIndexImpl implements EntityIndex {
 
         AdminClient admin = client.admin();
         try {
-            CreateIndexResponse r = admin.indices().prepareCreate(indexName).execute().actionGet();
-            log.debug("Created new Index Name [{}] ACK=[{}]", indexName, r.isAcknowledged());
+            CreateIndexResponse cir = admin.indices().prepareCreate(indexName).execute().actionGet();
+            log.debug("Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged());
 
             client.admin().indices().prepareRefresh(indexName).execute().actionGet();
 
+            XContentBuilder xcb = IndexingUtils.createDoubleStringIndexMapping( 
+                XContentFactory.jsonBuilder(), "_default_");
+
+            PutIndexTemplateResponse pitr = client.admin().indices()
+                .preparePutTemplate("usergrid_template")
+                .setTemplate(config.getIndexPrefix() + "*")
+                .addMapping("_default_", xcb)
+                .execute()
+                .actionGet();
+
+            log.debug("Create Mapping for new Index Name [{}] ACK=[{}]", 
+                    indexName, pitr.isAcknowledged());
+
             try {
                 // TODO: figure out what refresh above is not enough to ensure index is ready
                 Thread.sleep(500);
-            } catch (InterruptedException ex) {
-            }
-        } catch (IndexAlreadyExistsException ignored) {
-            // expected
+            } catch (InterruptedException ex) {}
+
+        } catch (IndexAlreadyExistsException expected) {
+            // this is expected to happen if index already exists
+
+        } catch ( IOException ioe ) {
+            throw new RuntimeException("Error setting up index", ioe);
         }
     }
 
@@ -135,7 +147,7 @@ public class EsEntityIndexImpl implements EntityIndex {
     @Override
     public CandidateResults search(final IndexScope indexScope, final Query query) {
 
-        final String indexType = createCollectionScopeTypeName(indexScope);
+        final String indexType = IndexingUtils.createCollectionScopeTypeName(indexScope);
 
         QueryBuilder qb = query.createQueryBuilder();
 
@@ -203,7 +215,8 @@ public class EsEntityIndexImpl implements EntityIndex {
             }
             log.debug("Executing query with cursor: {} ", scrollId);
 
-            SearchScrollRequestBuilder ssrb = client.prepareSearchScroll(scrollId).setScroll(cursorTimeout
+ "m");
+            SearchScrollRequestBuilder ssrb = 
+                    client.prepareSearchScroll(scrollId).setScroll(cursorTimeout + "m");
             searchResponse = ssrb.execute().actionGet();
         }
 
@@ -234,70 +247,6 @@ public class EsEntityIndexImpl implements EntityIndex {
         return candidateResults;
     }
 
-    /**
-     * Build mappings for data to be indexed. Setup String fields as not_analyzed and analyzed,
-     * where the analyzed field is named {name}_ug_analyzed
-     *
-     * @param builder Add JSON object to this builder.
-     * @param type ElasticSearch type of entity.
-     *
-     * @return Content builder with JSON for mapping.
-     *
-     * @throws java.io.IOException On JSON generation error.
-     */
-    public static XContentBuilder createDoubleStringIndexMapping( XContentBuilder builder,
String type )
-            throws IOException {
-
-        builder = builder
-
-            .startObject()
-
-                .startObject( type )
-
-                    .startArray( "dynamic_templates" )
-
-                        // any string with field name that starts with sa_ gets analyzed
-                        .startObject()
-                            .startObject( "template_1" )
-                                .field( "match", ANALYZED_STRING_PREFIX + "*" )
-                                .field( "match_mapping_type", "string" )
-                                .startObject( "mapping" ).field( "type", "string" )
-                                    .field( "index", "analyzed" )
-                                .endObject()
-                            .endObject()
-                        .endObject()
-
-                            // all other strings are not analyzed
-                        .startObject()
-                            .startObject( "template_2" )
-                                .field( "match", "*" )
-                                .field( "match_mapping_type", "string" )
-                                .startObject( "mapping" )
-                                    .field( "type", "string" )
-                                    .field( "index", "not_analyzed" )
-                                .endObject()
-                            .endObject()
-                        .endObject()
-
-                        // fields names starting with go_ get geo-indexed
-                        .startObject()
-                            .startObject( "template_3" )
-                                .field( "match", GEO_PREFIX + "location" )
-                                .startObject( "mapping" )
-                                    .field( "type", "geo_point" )
-                                .endObject()
-                            .endObject()
-                        .endObject()
-
-                    .endArray()
-
-                .endObject()
-
-            .endObject();
-
-        return builder;
-    }
-
     public void refresh() {
         client.admin().indices().prepareRefresh(indexName).execute().actionGet();
         log.debug("Refreshed index: " + indexName);

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c0edbe19/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 8dbaa0d..f3e1ba2 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -18,12 +18,14 @@ package org.apache.usergrid.persistence.index.impl;/*
  */
 
 
+import java.io.IOException;
 import java.util.UUID;
 
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.index.IndexScope;
 import org.apache.usergrid.persistence.model.entity.Entity;
 import org.apache.usergrid.persistence.model.entity.Id;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 
 
 public class IndexingUtils {
@@ -104,4 +106,68 @@ public class IndexingUtils {
     }
 
 
+    /**
+     * Build mappings for data to be indexed. Setup String fields as not_analyzed and analyzed,
+     * where the analyzed field is named {name}_ug_analyzed
+     *
+     * @param builder Add JSON object to this builder.
+     * @param type ElasticSearch type of entity.
+     *
+     * @return Content builder with JSON for mapping.
+     *
+     * @throws java.io.IOException On JSON generation error.
+     */
+    public static XContentBuilder createDoubleStringIndexMapping( 
+            XContentBuilder builder, String type ) throws IOException {
+
+        builder = builder
+
+            .startObject()
+
+                .startObject( type )
+
+                    .startArray( "dynamic_templates" )
+
+                        // any string with field name that starts with sa_ gets analyzed
+                        .startObject()
+                            .startObject( "template_1" )
+                                .field( "match", ANALYZED_STRING_PREFIX + "*" )
+                                .field( "match_mapping_type", "string" )
+                                .startObject( "mapping" ).field( "type", "string" )
+                                    .field( "index", "analyzed" )
+                                .endObject()
+                            .endObject()
+                        .endObject()
+
+                            // all other strings are not analyzed
+                        .startObject()
+                            .startObject( "template_2" )
+                                .field( "match", "*" )
+                                .field( "match_mapping_type", "string" )
+                                .startObject( "mapping" )
+                                    .field( "type", "string" )
+                                    .field( "index", "not_analyzed" )
+                                .endObject()
+                            .endObject()
+                        .endObject()
+
+                        // fields names starting with go_ get geo-indexed
+                        .startObject()
+                            .startObject( "template_3" )
+                                .field( "match", GEO_PREFIX + "location" )
+                                .startObject( "mapping" )
+                                    .field( "type", "geo_point" )
+                                .endObject()
+                            .endObject()
+                        .endObject()
+
+                    .endArray()
+
+                .endObject()
+
+            .endObject();
+
+        return builder;
+    }
+
 }


Mime
View raw message