usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sfeld...@apache.org
Subject [06/11] incubator-usergrid git commit: Addes Cache to MapMangerFactorImpl for efficiency
Date Tue, 10 Mar 2015 21:12:05 GMT
Addes Cache to MapMangerFactorImpl for efficiency

Adds timeout for string storage in our map

Adds time UUID generated cursors to avoid USERGRID-461.  Note that the map CF will need to
be tuned appropriately.  I.E  lower gc_grace times since columns are expire and are never
deleted.  SSD's with key cache, or row+ key cache for spinning disks at the physical tier.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5d95ebba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5d95ebba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5d95ebba

Branch: refs/heads/USERGRID-405
Commit: 5d95ebbadc37609c200ea8def435b945c9641bac
Parents: e27f090
Author: Todd Nine <tnine@apigee.com>
Authored: Mon Mar 9 16:14:21 2015 -0600
Committer: Todd Nine <tnine@apigee.com>
Committed: Mon Mar 9 16:14:21 2015 -0600

----------------------------------------------------------------------
 .../persistence/map/MapManagerFactory.java      |   2 +
 .../persistence/map/guice/MapModule.java        |   6 +-
 .../map/impl/MapManagerFactoryImpl.java         |  67 ++++++++++
 stack/corepersistence/queryindex/pom.xml        |  16 ++-
 .../persistence/index/guice/IndexModule.java    |   4 +
 .../index/impl/EsEntityIndexFactoryImpl.java    |  12 +-
 .../index/impl/EsEntityIndexImpl.java           |  57 +++++++--
 .../persistence/index/impl/IndexingUtils.java   |   2 +-
 .../persistence/index/utils/StringUtils.java    | 126 ++-----------------
 .../persistence/index/utils/UUIDUtils.java      |   4 -
 .../persistence/index/impl/EntityIndexTest.java |  99 ++++++++++++++-
 11 files changed, 247 insertions(+), 148 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d95ebba/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManagerFactory.java
b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManagerFactory.java
index a60cdfc..81531c9 100644
--- a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManagerFactory.java
+++ b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManagerFactory.java
@@ -27,4 +27,6 @@ public interface MapManagerFactory {
      * Get the map manager
      */
     public MapManager createMapManager( final MapScope scope );
+
+    void invalidate();
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d95ebba/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/guice/MapModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/guice/MapModule.java
b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/guice/MapModule.java
index d167151..5c46c87 100644
--- a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/guice/MapModule.java
+++ b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/guice/MapModule.java
@@ -21,6 +21,7 @@ package org.apache.usergrid.persistence.map.guice;
 import org.apache.usergrid.persistence.core.migration.schema.Migration;
 import org.apache.usergrid.persistence.map.MapManager;
 import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.map.impl.MapManagerFactoryImpl;
 import org.apache.usergrid.persistence.map.impl.MapManagerImpl;
 import org.apache.usergrid.persistence.map.impl.MapSerialization;
 import org.apache.usergrid.persistence.map.impl.MapSerializationImpl;
@@ -43,12 +44,11 @@ public class MapModule extends AbstractModule {
     protected void configure() {
 
         // create a guice factory for getting our collection manager
-        install( new FactoryModuleBuilder().implement( MapManager.class, MapManagerImpl.class
)
-                                           .build( MapManagerFactory.class ) );
-
+        bind(MapManagerFactory.class).to( MapManagerFactoryImpl.class );
 
         bind( MapSerialization.class).to( MapSerializationImpl.class );
 
+
         Multibinder<Migration> migrationBinding = Multibinder.newSetBinder( binder(),
Migration.class );
         migrationBinding.addBinding().to(  Key.get( MapSerialization.class ) );
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d95ebba/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapManagerFactoryImpl.java
b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapManagerFactoryImpl.java
new file mode 100644
index 0000000..e69a02e
--- /dev/null
+++ b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapManagerFactoryImpl.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.persistence.map.impl;
+
+import com.google.common.base.Preconditions;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.netflix.astyanax.Execution;
+import org.apache.usergrid.persistence.map.MapManager;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.map.MapScope;
+
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Returns map managers, built to handle caching
+ */
+@Singleton
+public class MapManagerFactoryImpl implements MapManagerFactory {
+    private final MapSerialization mapSerialization;
+    private LoadingCache<MapScope, MapManager> mmCache =
+        CacheBuilder.newBuilder().maximumSize( 1000 ).build( new CacheLoader<MapScope,
MapManager>() {
+            public MapManager load( MapScope scope ) {
+                return  new MapManagerImpl(scope,mapSerialization);
+            }
+        } );
+
+    @Inject
+    public MapManagerFactoryImpl(final MapSerialization mapSerialization){
+
+        this.mapSerialization = mapSerialization;
+    }
+
+    @Override
+    public MapManager createMapManager(MapScope scope) {
+        Preconditions.checkNotNull(scope);
+        try{
+            return mmCache.get(scope);
+        }catch (ExecutionException ee){
+            throw new RuntimeException(ee);
+        }
+    }
+
+    @Override
+    public void invalidate() {
+        mmCache.invalidateAll();
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d95ebba/stack/corepersistence/queryindex/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/pom.xml b/stack/corepersistence/queryindex/pom.xml
index 8604316..f6ae718 100644
--- a/stack/corepersistence/queryindex/pom.xml
+++ b/stack/corepersistence/queryindex/pom.xml
@@ -50,10 +50,10 @@
                 <artifactId>chop-maven-plugin</artifactId>
                 <version>${chop.version}</version>
 
-                
+
                 NOTE: you should be putting most of these variables into your settings.xml
                 as an automatically activated profile.
-                
+
 
                 <configuration>
                     <accessKey>${aws.s3.key}</accessKey>
@@ -74,11 +74,11 @@
                     <runnerKeyPairName>${runner.keypair.name}</runnerKeyPairName>
                     <runnerCount>6</runnerCount>
                     <securityGroupExceptions>
-                        
+
                         Add your own IP address as an exception to allow access
                         but please do this in the settings.xml file .. essentially
                         all parameters should be in the settings.xml file.
-                        
+
                         <param>${myip.address}/32:24981</param>
                         <param>${myip.address}/32:22</param>
                     </securityGroupExceptions>
@@ -99,6 +99,14 @@
             <type>jar</type>
         </dependency>
 
+
+        <dependency>
+            <groupId>${project.parent.groupId}</groupId>
+            <artifactId>map</artifactId>
+            <version>${project.version}</version>
+            <type>jar</type>
+        </dependency>
+
         <dependency>
             <groupId>org.elasticsearch</groupId>
             <artifactId>elasticsearch</artifactId>

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d95ebba/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index 6fee17e..ebd9098 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -27,6 +27,8 @@ import org.apache.usergrid.persistence.index.impl.EsEntityIndexFactoryImpl;
 import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
 import org.apache.usergrid.persistence.index.impl.EsIndexBufferConsumerImpl;
 import org.apache.usergrid.persistence.index.impl.EsIndexBufferProducerImpl;
+import org.apache.usergrid.persistence.map.guice.MapModule;
+
 import org.safehaus.guicyfig.GuicyFigModule;
 
 
@@ -38,6 +40,8 @@ public class IndexModule extends AbstractModule {
         // install our configuration
         install(new GuicyFigModule(IndexFig.class));
 
+        install(new MapModule());
+
 
         bind(EntityIndexFactory.class).to( EsEntityIndexFactoryImpl.class );
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d95ebba/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
index 3a9f790..8af309d 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
@@ -23,7 +23,6 @@ import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
 
 import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -31,6 +30,7 @@ import org.apache.usergrid.persistence.index.EntityIndex;
 import org.apache.usergrid.persistence.index.EntityIndexFactory;
 import org.apache.usergrid.persistence.index.IndexBufferProducer;
 import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
 
 import java.util.concurrent.ExecutionException;
 
@@ -44,23 +44,29 @@ public class EsEntityIndexFactoryImpl implements EntityIndexFactory{
     private final EsIndexCache indexCache;
     private final IndexBufferProducer indexBatchBufferProducer;
     private final MetricsFactory metricsFactory;
+    private final MapManagerFactory mapManagerFactory;
+    private final IndexFig indexFig;
 
     private LoadingCache<ApplicationScope, EntityIndex> eiCache =
         CacheBuilder.newBuilder().maximumSize( 1000 ).build( new CacheLoader<ApplicationScope,
EntityIndex>() {
             public EntityIndex load( ApplicationScope scope ) {
-                return new EsEntityIndexImpl(scope,config, indexBatchBufferProducer, provider,indexCache,
metricsFactory);
+                return new EsEntityIndexImpl(scope,config, indexBatchBufferProducer, provider,indexCache,
metricsFactory,
+                    mapManagerFactory, indexFig );
             }
         } );
 
     @Inject
     public EsEntityIndexFactoryImpl( final IndexFig config, final EsProvider provider, final
EsIndexCache indexCache,
                                      final IndexBufferProducer indexBatchBufferProducer,
-                                     final MetricsFactory metricsFactory ){
+                                     final MetricsFactory metricsFactory, final MapManagerFactory
mapManagerFactory,
+                                     final IndexFig indexFig ){
         this.config = config;
         this.provider = provider;
         this.indexCache = indexCache;
         this.indexBatchBufferProducer = indexBatchBufferProducer;
         this.metricsFactory = metricsFactory;
+        this.mapManagerFactory = mapManagerFactory;
+        this.indexFig = indexFig;
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d95ebba/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index c9f5590..c92b299 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.persistence.index.impl;
 
 import com.codahale.metrics.Meter;
 import com.codahale.metrics.Timer;
+import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
@@ -38,6 +39,11 @@ import org.apache.usergrid.persistence.index.exceptions.IndexException;
 import org.apache.usergrid.persistence.index.query.CandidateResult;
 import org.apache.usergrid.persistence.index.query.CandidateResults;
 import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.persistence.index.utils.UUIDUtils;
+import org.apache.usergrid.persistence.map.MapManager;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.map.MapScope;
+import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
 import org.apache.usergrid.persistence.model.entity.Id;
 import org.apache.usergrid.persistence.model.entity.SimpleId;
 import org.apache.usergrid.persistence.model.util.UUIDGenerator;
@@ -81,6 +87,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.UUID;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.*;
@@ -99,6 +106,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
     private final IndexIdentifier.IndexAlias alias;
     private final IndexIdentifier indexIdentifier;
     private final IndexBufferProducer indexBatchBufferProducer;
+    private final IndexFig indexFig;
     private final Timer addTimer;
     private final Timer addWriteAliasTimer;
     private final Timer addReadAliasTimer;
@@ -141,14 +149,18 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
     private Timer allVersionsTimer;
     private Timer deletePreviousTimer;
 
+    private final MapManager mapManager;
+
 //    private final Timer indexTimer;
 
 
     @Inject
     public EsEntityIndexImpl( @Assisted final ApplicationScope appScope, final IndexFig config,
                               final IndexBufferProducer indexBatchBufferProducer, final EsProvider
provider,
-                              final EsIndexCache indexCache, final MetricsFactory metricsFactory)
{
+                              final EsIndexCache indexCache, final MetricsFactory metricsFactory,
+                              final MapManagerFactory mapManagerFactory, final IndexFig indexFig
) {
         this.indexBatchBufferProducer = indexBatchBufferProducer;
+        this.indexFig = indexFig;
         ValidationUtils.validateApplicationScope( appScope );
         this.applicationScope = appScope;
         this.esProvider = provider;
@@ -181,6 +193,10 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
             .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.all.versions.timer"
);
         this.deletePreviousTimer = metricsFactory
             .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.previous.versions.timer"
);
+
+        final MapScope mapScope = new MapScopeImpl( appScope.getApplication(), "cursorcache"
);
+
+        mapManager = mapManagerFactory.createMapManager( mapScope );
     }
 
     @Override
@@ -467,17 +483,26 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
             failureMonitor.success();
         }
         else {
-            String scrollId = query.getCursor();
-            if ( scrollId.startsWith( "\"" ) ) {
-                scrollId = scrollId.substring( 1 );
+            String userCursorString = query.getCursor();
+            if ( userCursorString.startsWith( "\"" ) ) {
+                userCursorString = userCursorString.substring( 1 );
             }
-            if ( scrollId.endsWith( "\"" ) ) {
-                scrollId = scrollId.substring( 0, scrollId.length() - 1 );
+            if ( userCursorString.endsWith( "\"" ) ) {
+                userCursorString = userCursorString.substring( 0, userCursorString.length()
- 1 );
             }
-            logger.debug( "Executing query with cursor: {} ", scrollId );
+
+            //now get the cursor from the map  and validate
+            final String esScrollCursor  = mapManager.getString( userCursorString );
+
+            Preconditions.checkArgument(esScrollCursor != null, "Could not find a cursor
for the value '{}' ",  esScrollCursor);
+
+
+
+            logger.debug( "Executing query with cursor: {} ", esScrollCursor );
+
 
             SearchScrollRequestBuilder ssrb = esProvider.getClient()
-                    .prepareSearchScroll(scrollId).setScroll( cursorTimeout + "m" );
+                    .prepareSearchScroll(esScrollCursor).setScroll( cursorTimeout + "m" );
 
             try {
                 //Added For Graphite Metrics
@@ -524,8 +549,20 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
         CandidateResults candidateResults = new CandidateResults( query, candidates );
 
         if ( candidates.size() >= query.getLimit() ) {
-            candidateResults.setCursor( searchResponse.getScrollId() );
-            logger.debug("   Cursor = " + searchResponse.getScrollId());
+            //USERGRID-461 our cursor is getting too large, map it to a new time UUID
+
+            final String userCursorString = org.apache.usergrid.persistence.index.utils.StringUtils.sanitizeUUID(
UUIDGenerator.newTimeUUID() );
+
+            final String esScrollCursor = searchResponse.getScrollId();
+
+            //now set this into our map module
+            final int minutes = indexFig.getQueryCursorTimeout();
+
+            //just truncate it, we'll never hit a long value anyway
+            mapManager.putString( userCursorString, esScrollCursor, ( int ) TimeUnit.MINUTES.toSeconds(
minutes ) );
+
+            candidateResults.setCursor( userCursorString );
+            logger.debug(" User cursor = {},  Cursor = {} ", userCursorString, esScrollCursor);
         }
 
         return candidateResults;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d95ebba/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index be0c96c..ffd98e9 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -152,7 +152,7 @@ public class IndexingUtils {
             .startObject()
 
                     /**  add routing  "_routing":{ "required":false,  "path":"ug_entityId"
**/
-                     //.startObject("_routing").field("required",true).field("path",ENTITYID_ID_FIELDNAME).endObject()
+                     .startObject("_routing").field("required",true).field("path",ENTITYID_ID_FIELDNAME).endObject()
                      .startArray("dynamic_templates")
                         // we need most specific mappings first since it's a stop on match
algorithm
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d95ebba/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/StringUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/StringUtils.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/StringUtils.java
index 8aabbbf..a567594 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/StringUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/StringUtils.java
@@ -18,6 +18,7 @@ package org.apache.usergrid.persistence.index.utils;
 
 
 import java.util.Arrays;
+import java.util.UUID;
 
 import org.apache.commons.io.IOUtils;
 
@@ -32,37 +33,6 @@ public class StringUtils extends org.apache.commons.lang.StringUtils {
     private static final Logger LOG = LoggerFactory.getLogger( StringUtils.class );
 
 
-    public static Object lower( Object obj ) {
-        if ( !( obj instanceof String ) ) {
-            return obj;
-        }
-        return ( ( String ) obj ).toLowerCase();
-    }
-
-
-    public static String stringOrSubstringAfterLast( String str, char c ) {
-        if ( str == null ) {
-            return null;
-        }
-        int i = str.lastIndexOf( c );
-        if ( i != -1 ) {
-            return str.substring( i + 1 );
-        }
-        return str;
-    }
-
-
-    public static String stringOrSubstringBeforeLast( String str, char c ) {
-        if ( str == null ) {
-            return null;
-        }
-        int i = str.lastIndexOf( c );
-        if ( i != -1 ) {
-            return str.substring( 0, i );
-        }
-        return str;
-    }
-
 
     public static String stringOrSubstringBeforeFirst( String str, char c ) {
         if ( str == null ) {
@@ -76,97 +46,17 @@ public class StringUtils extends org.apache.commons.lang.StringUtils {
     }
 
 
-    public static String stringOrSubstringAfterFirst( String str, char c ) {
-        if ( str == null ) {
-            return null;
-        }
-        int i = str.indexOf( c );
-        if ( i != -1 ) {
-            return str.substring( i + 1 );
-        }
-        return str;
-    }
-
-
-    public static String compactWhitespace( String str ) {
-        if ( str == null ) {
-            return null;
-        }
-        boolean prevWS = false;
-        StringBuilder builder = new StringBuilder();
-        for ( int i = 0; i < str.length(); i++ ) {
-            char c = str.charAt( i );
-            if ( Character.isWhitespace( c ) ) {
-                if ( !prevWS ) {
-                    builder.append( ' ' );
-                }
-                prevWS = true;
-            }
-            else {
-                prevWS = false;
-                builder.append( c );
-            }
-        }
-        return builder.toString().trim();
-    }
-
-
-    /** @return new string with replace applied */
-    public static String replaceAll( String source, String find, String replace ) {
-        if ( source == null ) {
-            return null;
-        }
-        while ( true ) {
-            String old = source;
-            source = source.replaceAll( find, replace );
-            if ( source.equals( old ) ) {
-                return source;
-            }
-        }
-    }
-
-
     public static String toString( Object obj ) {
         return string( obj );
     }
 
 
-    public static String toStringFormat( Object obj, String format ) {
-        if ( obj != null ) {
-            if ( format != null ) {
-                if ( obj.getClass().isArray() ) {
-                    return String.format( format, Arrays.toString( ( Object[] ) obj ) );
-                }
-                return String.format( format, string( obj ) );
-            }
-            else {
-                return string( obj );
-            }
-        }
-        return "";
-    }
-
-
-    public static boolean isString( Object obj ) {
-        return obj instanceof String;
-    }
-
-
-    public static boolean isStringOrNull( Object obj ) {
-        if ( obj == null ) {
-            return true;
-        }
-        return obj instanceof String;
-    }
-
-
-    public static String readClasspathFileAsString( String filePath ) {
-        try {
-            return IOUtils.toString( StringUtils.class.getResourceAsStream( filePath ) );
-        }
-        catch ( Exception e ) {
-            LOG.error( "Error getting file from classpath: " + filePath, e );
-        }
-        return null;
+    /**
+     * Remove dashes from our uuid
+     * @param uuid
+     * @return
+     */
+    public static String sanitizeUUID(final UUID uuid){
+        return uuid.toString().replace( "-", "" );
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d95ebba/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/UUIDUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/UUIDUtils.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/UUIDUtils.java
index fdffaef..b9b407b 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/UUIDUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/UUIDUtils.java
@@ -57,10 +57,6 @@ public class UUIDUtils {
 
     private static ReentrantLock tsLock = new ReentrantLock( true );
 
-    public static final UUID MIN_TIME_UUID = UUID.fromString( "00000000-0000-1000-8000-000000000000"
);
-
-    public static final UUID MAX_TIME_UUID = UUID.fromString( "ffffffff-ffff-1fff-bfff-ffffffffffff"
);
-
     public static final UUID ZERO_UUID = new UUID( 0, 0 );
 
     private static long timestampMillisNow = System.currentTimeMillis();

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d95ebba/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 70ae8c5..a3332a8 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -24,10 +24,7 @@ import java.util.*;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicLong;
 
-import org.apache.usergrid.persistence.core.future.BetterFuture;
 import org.apache.usergrid.persistence.index.*;
-import org.apache.usergrid.persistence.index.query.CandidateResult;
-import org.apache.usergrid.persistence.index.utils.IndexValidationUtils;
 import org.apache.usergrid.persistence.model.field.UUIDField;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -289,8 +286,8 @@ public class EntityIndexTest extends BaseIT {
         entityIndex.createBatch().index(indexScope , entity ).execute().get();
         entityIndex.refresh();
 
-        CandidateResults candidateResults = entityIndex.search( indexScope, SearchTypes.fromTypes(entity.getId().getType()),
-                Query.fromQL( "name contains 'Ferrari*'" ) );
+        CandidateResults candidateResults = entityIndex.search( indexScope,
+            SearchTypes.fromTypes( entity.getId().getType() ), Query.fromQL( "name contains
'Ferrari*'" ) );
         assertEquals( 1, candidateResults.size() );
 
         EntityIndexBatch batch = entityIndex.createBatch();
@@ -593,6 +590,98 @@ public class EntityIndexTest extends BaseIT {
         assertNotEquals( "cluster should be fine", Health.RED, ei.getIndexHealth() );
         assertNotEquals( "cluster should be ready now", Health.RED, ei.getClusterHealth()
);
     }
+
+
+    @Test
+    public void testCursorFormat() throws Exception {
+
+        Id appId = new SimpleId( "application" );
+        Id ownerId = new SimpleId( "owner" );
+
+        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+        IndexScope indexScope = new IndexScopeImpl( ownerId, "users" );
+
+
+        EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
+        entityIndex.initializeIndex();
+
+        final EntityIndexBatch batch = entityIndex.createBatch();
+
+
+        final int size = 10;
+
+        final List<Id> entities = new ArrayList<>( size );
+
+
+        for ( int i = 0; i < size; i++ ) {
+            final String middleName = "middleName" + UUIDUtils.newTimeUUID();
+            Map<String, Object> properties = new LinkedHashMap<String, Object>();
+            properties.put( "username", "edanuff" );
+            properties.put( "email", "ed@anuff.com" );
+            properties.put( "middlename", middleName );
+
+            Map entityMap = new HashMap() {{
+                put( "username", "edanuff" );
+                put( "email", "ed@anuff.com" );
+                put( "middlename", middleName );
+            }};
+
+            final Id userId = new SimpleId( "user" );
+
+            Entity user = EntityIndexMapUtils.fromMap( entityMap );
+            EntityUtils.setId( user, userId );
+            EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() );
+
+            user.setField( new UUIDField( IndexingUtils.ENTITYID_ID_FIELDNAME, UUIDGenerator.newTimeUUID()
) );
+
+            entities.add( userId );
+
+
+            batch.index( indexScope, user );
+        }
+
+
+        batch.execute().get();
+        entityIndex.refresh();
+
+
+        final int limit = 1;
+
+
+        final int expectedPages = size / limit;
+
+
+        String cursor = null;
+
+        for ( int i = 0; i < expectedPages; i++ ) {
+            //**
+            final Query query = Query.fromQL( "select *" );
+            query.setLimit( limit );
+
+            if ( cursor != null ) {
+                query.setCursor( cursor );
+            }
+
+            final CandidateResults results = entityIndex.search( indexScope, SearchTypes.allTypes(),
query );
+
+            assertTrue( results.hasCursor() );
+
+            cursor = results.getCursor();
+
+            assertEquals("Should be 16 bytes as hex", 32, cursor.length());
+
+
+
+
+            assertEquals( 1, results.size() );
+
+
+            assertEquals( results.get( 0 ).getId(), entities.get( i ) );
+        }
+    }
+
+
 }
 
 


Mime
View raw message