Return-Path: X-Original-To: apmail-usergrid-commits-archive@minotaur.apache.org Delivered-To: apmail-usergrid-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id BBAB1178DE for ; Tue, 10 Mar 2015 21:12:08 +0000 (UTC) Received: (qmail 59534 invoked by uid 500); 10 Mar 2015 21:12:00 -0000 Delivered-To: apmail-usergrid-commits-archive@usergrid.apache.org Received: (qmail 59471 invoked by uid 500); 10 Mar 2015 21:12:00 -0000 Mailing-List: contact commits-help@usergrid.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@usergrid.incubator.apache.org Delivered-To: mailing list commits@usergrid.incubator.apache.org Received: (qmail 59268 invoked by uid 99); 10 Mar 2015 21:12:00 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 10 Mar 2015 21:12:00 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 4BF2AE1882; Tue, 10 Mar 2015 21:12:00 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sfeldman@apache.org To: commits@usergrid.apache.org Date: Tue, 10 Mar 2015 21:12:09 -0000 Message-Id: In-Reply-To: <4e1ec283e31941d98e796fb6abfb68c0@git.apache.org> References: <4e1ec283e31941d98e796fb6abfb68c0@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [10/11] incubator-usergrid git commit: fix for list types changing from string to int or whatever fix for list types changing from string to int or whatever Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e5cab76f Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e5cab76f Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e5cab76f Branch: refs/heads/USERGRID-405 Commit: e5cab76f8b8fc1e4a60ea23274e7a8a0cc79f759 Parents: ce14542 Author: Shawn Feldman Authored: Tue Mar 10 15:01:54 2015 -0600 Committer: Shawn Feldman Committed: Tue Mar 10 15:01:54 2015 -0600 ---------------------------------------------------------------------- .../index/impl/EsEntityIndexBatchImpl.java | 14 +-- .../persistence/index/impl/IndexingUtils.java | 109 ++++++++++--------- .../persistence/index/impl/EntityIndexTest.java | 38 ++++++- 3 files changed, 91 insertions(+), 70 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e5cab76f/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java index d987b29..18cc4fc 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java @@ -56,15 +56,7 @@ import com.codahale.metrics.Timer; import rx.Observable; import rx.functions.Func1; -import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX; -import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX; -import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_ID_FIELDNAME; -import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_CONTEXT_FIELDNAME; -import static org.apache.usergrid.persistence.index.impl.IndexingUtils.GEO_PREFIX; -import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX; -import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX; -import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName; -import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexDocId; +import static org.apache.usergrid.persistence.index.impl.IndexingUtils.*; public class EsEntityIndexBatchImpl implements EntityIndexBatch { @@ -255,7 +247,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch { if ( f instanceof ListField ) { List list = ( List ) field.getValue(); - entityMap.put( field.getName().toLowerCase(), + entityMap.put(LIST_PREFIX + field.getName().toLowerCase(), new ArrayList( processCollectionForMap( list ) ) ); if ( !list.isEmpty() ) { @@ -267,7 +259,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch { } else if ( f instanceof ArrayField ) { List list = ( List ) field.getValue(); - entityMap.put( field.getName().toLowerCase(), + entityMap.put(ARRAY_PREFIX + field.getName().toLowerCase(), new ArrayList( processCollectionForMap( list ) ) ); } else if ( f instanceof SetField ) { http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e5cab76f/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java index ffd98e9..41a4680 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java @@ -34,6 +34,9 @@ public class IndexingUtils { public static final String STRING_PREFIX = "su_"; public static final String ANALYZED_STRING_PREFIX = "sa_"; + public static final String ARRAY_PREFIX = "ar_"; + public static final String LIST_PREFIX = "li_"; + public static final String GEO_PREFIX = "go_"; public static final String NUMBER_PREFIX = "nu_"; public static final String BOOLEAN_PREFIX = "bu_"; @@ -153,75 +156,75 @@ public class IndexingUtils { /** add routing "_routing":{ "required":false, "path":"ug_entityId" **/ .startObject("_routing").field("required",true).field("path",ENTITYID_ID_FIELDNAME).endObject() - .startArray("dynamic_templates") + .startArray("dynamic_templates") // we need most specific mappings first since it's a stop on match algorithm - .startObject() + .startObject() - .startObject( "entity_id_template" ) - .field( "match", IndexingUtils.ENTITYID_ID_FIELDNAME ) - .field( "match_mapping_type", "string" ) - .startObject( "mapping" ).field( "type", "string" ) - .field( "index", "not_analyzed" ) - .endObject() - .endObject() - .endObject() + .startObject("entity_id_template" ) + .field("match", IndexingUtils.ENTITYID_ID_FIELDNAME ) + .field("match_mapping_type", "string" ) + .startObject( "mapping").field("type", "string" ) + .field("index", "not_analyzed" ) + .endObject() + .endObject() + .endObject() - .startObject() - .startObject( "entity_context_template" ) + .startObject() + .startObject("entity_context_template" ) .field( "match", IndexingUtils.ENTITY_CONTEXT_FIELDNAME ) - .field( "match_mapping_type", "string" ) - .startObject( "mapping" ) + .field("match_mapping_type", "string" ) + .startObject( "mapping" ) .field( "type", "string" ) - .field( "index", "not_analyzed" ).endObject() - .endObject() - .endObject() - - .startObject() - .startObject( "entity_version_template" ) - .field( "match", IndexingUtils.ENTITY_VERSION_FIELDNAME ) - .field( "match_mapping_type", "string" ) - .startObject( "mapping" ).field( "type", "long" ) - .endObject() - .endObject() - .endObject() + .field("index", "not_analyzed").endObject() + .endObject() + .endObject() + + .startObject() + .startObject("entity_version_template" ) + .field("match", IndexingUtils.ENTITY_VERSION_FIELDNAME ) + .field("match_mapping_type", "string" ) + .startObject( "mapping").field("type", "long" ) + .endObject() + .endObject() + .endObject() // any string with field name that starts with sa_ gets analyzed - .startObject() - .startObject( "template_1" ) + .startObject() + .startObject("template_1" ) .field( "match", ANALYZED_STRING_PREFIX + "*" ) - .field( "match_mapping_type", "string" ).startObject( "mapping" ) + .field("match_mapping_type", "string" ).startObject( "mapping" ) .field( "type", "string" ) - .field( "index", "analyzed" ) - .endObject() - .endObject() + .field("index", "analyzed" ) + .endObject() + .endObject() - .endObject() + .endObject() // all other strings are not analyzed - .startObject() - .startObject( "template_2" ) + .startObject() + .startObject("template_2" ) //todo, should be string prefix, remove 2 field mapping - .field( "match", "*" ) - .field( "match_mapping_type", "string" ) - .startObject( "mapping" ) - .field( "type", "string" ) - .field( "index", "not_analyzed" ) - .endObject() - .endObject() - .endObject() + .field("match", "*" ) + .field("match_mapping_type", "string" ) + .startObject( "mapping" ) + .field("type", "string" ) + .field("index", "not_analyzed" ) + .endObject() + .endObject() + .endObject() // fields names starting with go_ get geo-indexed - .startObject() - .startObject( "template_3" ) - .field( "match", GEO_PREFIX + "location" ) - .startObject( "mapping" ) - .field( "type", "geo_point" ) - .endObject() - .endObject() - .endObject() - - .endArray() + .startObject() + .startObject("template_3" ) + .field("match", GEO_PREFIX + "location" ) + .startObject( "mapping" ) + .field("type", "geo_point" ) + .endObject() + .endObject() + .endObject() + + .endArray() .endObject(); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e5cab76f/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java index a15053c..e4043d2 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java @@ -25,9 +25,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicLong; import org.apache.usergrid.persistence.index.*; -import org.apache.usergrid.persistence.model.field.ArrayField; -import org.apache.usergrid.persistence.model.field.EntityObjectField; -import org.apache.usergrid.persistence.model.field.UUIDField; +import org.apache.usergrid.persistence.model.field.*; import org.apache.usergrid.persistence.model.field.value.EntityObject; import org.junit.Ignore; import org.junit.Test; @@ -49,7 +47,6 @@ import org.apache.usergrid.persistence.index.utils.UUIDUtils; import org.apache.usergrid.persistence.model.entity.Entity; import org.apache.usergrid.persistence.model.entity.Id; import org.apache.usergrid.persistence.model.entity.SimpleId; -import org.apache.usergrid.persistence.model.field.StringField; import org.apache.usergrid.persistence.model.util.UUIDGenerator; import com.fasterxml.jackson.core.type.TypeReference; @@ -90,7 +87,7 @@ public class EntityIndexTest extends BaseIT { } @Test - @Ignore("this is a problem i will work on when i can breathe") +// @Ignore("this is a problem i will work on when i can breathe") public void testIndexVariations() throws IOException { Id appId = new SimpleId( "application" ); @@ -110,6 +107,8 @@ public class EntityIndexTest extends BaseIT { batch.index(indexScope, entity); batch.execute().get(); + entity = new Entity( entityType ); + entity.setField(new UUIDField(IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID())); EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID()); List list = new ArrayList<>(); list.add("test"); @@ -117,6 +116,9 @@ public class EntityIndexTest extends BaseIT { batch.index(indexScope, entity); batch.execute().get(); + + entity = new Entity( entityType ); + entity.setField(new UUIDField(IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID())); EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID()); EntityObject testObj = new EntityObject(); testObj.setField(new StringField("test","testFiedl")); @@ -124,9 +126,32 @@ public class EntityIndexTest extends BaseIT { batch.index(indexScope, entity); batch.execute().get(); + entity = new Entity( entityType ); + entity.setField(new UUIDField(IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID())); + EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID()); + List listint = new ArrayList<>(); + listint.add(0); + entity.setField(new ArrayField("testfield", listint)); + batch.index(indexScope, entity); + batch.execute().get(); + + entity = new Entity( entityType ); + entity.setField(new UUIDField(IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID())); + EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID()); + List listObj = new ArrayList<>(); + EntityObject listObjField = new EntityObject(); + listObjField.setField(new StringField("testasf","somevalue")); + listObj.add(listObjField); + listObjField = new EntityObject(); + listObjField.setField(new IntegerField("testasf",0)); + listObj.add(listObjField); + entity.setField(new ArrayField("testfield", listObj)); + batch.index(indexScope, entity); + batch.execute().get(); + entityIndex.refresh(); + testQuery(indexScope, searchTypes, entityIndex, "select *", 5); - testQueries( indexScope, searchTypes, entityIndex ); } @Test @@ -254,6 +279,7 @@ public class EntityIndexTest extends BaseIT { entityIndex.refresh(); //Hilda Youn + testQuery(indexScope, searchTypes, entityIndex, "name = 'Bowers Oneil'", 0); }