usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mru...@apache.org
Subject usergrid git commit: Fix issue with map manager key sorting. Enhance buffer sizing by using byte length of string components.
Date Fri, 19 Aug 2016 06:55:32 GMT
Repository: usergrid
Updated Branches:
  refs/heads/master 92d5f4076 -> fcbe80367


Fix issue with map manager key sorting.  Enhance buffer sizing by using byte length of string
components.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/fcbe8036
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/fcbe8036
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/fcbe8036

Branch: refs/heads/master
Commit: fcbe803679ab4832a9ec2b3d26fd35eaec61202d
Parents: 92d5f40
Author: Michael Russo <mrusso@apigee.com>
Authored: Thu Aug 18 23:55:30 2016 -0700
Committer: Michael Russo <mrusso@apigee.com>
Committed: Thu Aug 18 23:55:30 2016 -0700

----------------------------------------------------------------------
 .../impl/ScopedCacheSerializationImpl.java      |  2 +-
 .../UniqueValueSerializationStrategyV1Impl.java | 23 ++++++----
 .../UniqueValueSerializationStrategyV2Impl.java | 20 ++++----
 .../usergrid/persistence/map/MapManager.java    |  2 +-
 .../map/impl/MapSerializationImpl.java          |  4 +-
 .../persistence/map/MapManagerTest.java         | 48 +++++++++++++++++---
 6 files changed, 70 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcbe8036/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java
b/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java
index 1334650..2a44f2b 100644
--- a/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java
+++ b/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java
@@ -303,7 +303,7 @@ public class ScopedCacheSerializationImpl<K,V> implements ScopedCacheSerializati
         keys.add(3, rowKeyString);
 
         // UUIDs are 16 bytes, allocate the buffer accordingly
-        int size = 16+ownerType.length()+rowKeyString.length();
+        int size = 16+ownerType.getBytes().length+rowKeyString.getBytes().length;
 
         // ints are 4 bytes, add for the bucket
         size += 4;

http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcbe8036/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java
b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java
index 55ba011..1435c1e 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java
@@ -198,14 +198,15 @@ public class UniqueValueSerializationStrategyV1Impl  extends UniqueValueSerializ
 
         String comparator = UUID_TYPE_REVERSED;
 
-        int size = 16+fieldEntry.getField().getName().length()+fieldEntry.getField().getValue().toString().length()+
-            fieldEntry.getField().getTypeName().name().length();
+        int size = 16+fieldEntry.getField().getName().getBytes().length
+            +fieldEntry.getField().getValue().toString().getBytes().length+
+            fieldEntry.getField().getTypeName().name().getBytes().length;
 
         // we always need to add length for the 2 byte comparator short,  2 byte length short
and 1 byte equality
         size += keys.size()*5;
 
         // uuid type comparator is longest, ensure we allocate buffer using the max size
to avoid overflow
-        size += keys.size()*comparator.length();
+        size += keys.size()*comparator.getBytes().length;
 
         ByteBuffer stuff = ByteBuffer.allocate(size);
 
@@ -228,7 +229,7 @@ public class UniqueValueSerializationStrategyV1Impl  extends UniqueValueSerializ
                 stuff.putShort((short) ('耀' | a));
             }else{
                 comparator = "UTF8Type"; // only strings are being serialized other than
UUIDs here
-                stuff.putShort((short)comparator.length());
+                stuff.putShort((short)comparator.getBytes().length);
                 stuff.put(DataType.serializeValue(comparator, ProtocolVersion.NEWEST_SUPPORTED));
             }
 
@@ -293,13 +294,13 @@ public class UniqueValueSerializationStrategyV1Impl  extends UniqueValueSerializ
         keys.add(entityVersion.getEntityId().getType());
 
         // UUIDs are 16 bytes
-        int size = 16+16+entityVersion.getEntityId().getType().length();
+        int size = 16+16+entityVersion.getEntityId().getType().getBytes().length;
 
         // we always need to add length for the 2 byte comparator short,  2 byte length short
and 1 byte equality
         size += keys.size()*5;
 
         // we always add comparator to the buffer as well
-        size += keys.size()*comparator.length();
+        size += keys.size()*comparator.getBytes().length;
 
         ByteBuffer stuff = ByteBuffer.allocate(size);
 
@@ -311,7 +312,7 @@ public class UniqueValueSerializationStrategyV1Impl  extends UniqueValueSerializ
                 comparator = "UTF8Type"; // if it's not a UUID, the only other thing we're
serializing is text
             }
 
-            stuff.putShort((short)comparator.length());
+            stuff.putShort((short)comparator.getBytes().length);
             stuff.put(DataType.serializeValue(comparator, ProtocolVersion.NEWEST_SUPPORTED));
 
             ByteBuffer kb = DataType.serializeValue(key, ProtocolVersion.NEWEST_SUPPORTED);
@@ -457,8 +458,9 @@ public class UniqueValueSerializationStrategyV1Impl  extends UniqueValueSerializ
 
 
         // UUIDs are 16 bytes, allocate the buffer accordingly
-        int size = 16 + applicationType.length() + 16 + applicationType.length() + collectionName.length()
+
-            fieldType.length() + fieldName.length()+fieldValueString.length();
+        int size = 16 + applicationType.getBytes().length + 16 + applicationType.getBytes().length
+
+            collectionName.getBytes().length + fieldType.getBytes().length + fieldName.getBytes().length
+            + fieldValueString.getBytes().length;
 
 
         // we always need to add length for the 2 byte short and 1 byte equality
@@ -503,7 +505,8 @@ public class UniqueValueSerializationStrategyV1Impl  extends UniqueValueSerializ
         keys.add(entityId);
         keys.add(entityType);
 
-        int size = 16+applicationType.length()+16+applicationType.length()+collectionName.length()+16+entityType.length();
+        int size = 16+applicationType.getBytes().length+16+applicationType.getBytes().length
+            +collectionName.getBytes().length+16+entityType.getBytes().length;
 
         // we always need to add length for the 2 byte short and 1 byte equality
         size += keys.size()*3;

http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcbe8036/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java
b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java
index a5fceeb..518937d 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java
@@ -187,14 +187,15 @@ public class UniqueValueSerializationStrategyV2Impl  extends UniqueValueSerializ
 
         String comparator = UUID_TYPE_REVERSED;
 
-        int size = 16+fieldEntry.getField().getName().length()+fieldEntry.getField().getValue().toString().length()+
-            fieldEntry.getField().getTypeName().name().length();
+        int size = 16+fieldEntry.getField().getName().getBytes().length
+            +fieldEntry.getField().getValue().toString().getBytes().length+
+            fieldEntry.getField().getTypeName().name().getBytes().length;
 
         // we always need to add length for the 2 byte comparator short,  2 byte length short
and 1 byte equality
         size += keys.size()*5;
 
         // uuid type comparator is longest, ensure we allocate buffer using the max size
to avoid overflow
-        size += keys.size()*comparator.length();
+        size += keys.size()*comparator.getBytes().length;
 
         ByteBuffer stuff = ByteBuffer.allocate(size);
 
@@ -217,7 +218,7 @@ public class UniqueValueSerializationStrategyV2Impl  extends UniqueValueSerializ
                 stuff.putShort((short) ('耀' | a));
             }else{
                 comparator = "UTF8Type"; // only strings are being serialized other than
UUIDs here
-                stuff.putShort((short)comparator.length());
+                stuff.putShort((short)comparator.getBytes().length);
                 stuff.put(DataType.serializeValue(comparator, ProtocolVersion.NEWEST_SUPPORTED));
             }
 
@@ -282,13 +283,13 @@ public class UniqueValueSerializationStrategyV2Impl  extends UniqueValueSerializ
         keys.add(entityVersion.getEntityId().getType());
 
         // UUIDs are 16 bytes
-        int size = 16+16+entityVersion.getEntityId().getType().length();
+        int size = 16+16+entityVersion.getEntityId().getType().getBytes().length;
 
         // we always need to add length for the 2 byte comparator short,  2 byte length short
and 1 byte equality
         size += keys.size()*5;
 
         // we always add comparator to the buffer as well
-        size += keys.size()*comparator.length();
+        size += keys.size()*comparator.getBytes().length;
 
         ByteBuffer stuff = ByteBuffer.allocate(size);
 
@@ -303,7 +304,7 @@ public class UniqueValueSerializationStrategyV2Impl  extends UniqueValueSerializ
                 comparator = "UTF8Type"; // if it's not a UUID, the only other thing we're
serializing is text
             }
 
-            stuff.putShort((short)comparator.length());
+            stuff.putShort((short)comparator.getBytes().length);
             stuff.put(DataType.serializeValue(comparator, ProtocolVersion.NEWEST_SUPPORTED));
 
             ByteBuffer kb = DataType.serializeValue(key, ProtocolVersion.NEWEST_SUPPORTED);
@@ -436,7 +437,8 @@ public class UniqueValueSerializationStrategyV2Impl  extends UniqueValueSerializ
 
 
         // UUIDs are 16 bytes, allocate the buffer accordingly
-        int size = 16 + applicationType.length() + entityType.length() + fieldType.length()
+ fieldName.length()+fieldValueString.length();
+        int size = 16 + applicationType.getBytes().length + entityType.getBytes().length
+            + fieldType.getBytes().length + fieldName.getBytes().length+fieldValueString.getBytes().length;
 
 
         // we always need to add length for the 2 byte short and 1 byte equality
@@ -470,7 +472,7 @@ public class UniqueValueSerializationStrategyV2Impl  extends UniqueValueSerializ
         keys.add(entityId);
         keys.add(entityType);
 
-        int size = 16+applicationType.length()+16+entityType.length();
+        int size = 16+applicationType.getBytes().length+16+entityType.getBytes().length;
 
         // we always need to add length for the 2 byte short and 1 byte equality
         size += keys.size()*3;

http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcbe8036/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java
b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java
index e1d121e..bd1f444 100644
--- a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java
+++ b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java
@@ -92,7 +92,7 @@ public interface MapManager {
     void delete( final String key );
 
     /**
-     * Return a page of keys that exist within the map.  Keys are sorted descending.
+     * Return a page of keys that exist within the map.  Keys are sorted ascending in lexicographical
ordering.
      * @param cursor
      * @param limit
      */

http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcbe8036/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
index 282974c..2c77ebf64 100644
--- a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
+++ b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
@@ -70,7 +70,7 @@ public class MapSerializationImpl implements MapSerialization {
             put( "column1", DataType.Name.BLOB );
             put( "value", DataType.Name.BLOB ); }};
     private static final Map<String, String> MAP_KEYS_CLUSTERING_ORDER =
-        new HashMap<String, String>(){{ put( "column1", "DESC" ); }};
+        new HashMap<String, String>(){{ put( "column1", "ASC" ); }};
 
 
 
@@ -465,7 +465,7 @@ public class MapSerializationImpl implements MapSerialization {
         }
 
         // UUIDs are 16 bytes, allocate the buffer accordingly
-        int size = 16+ownerType.length()+mapName.length()+mapKey.length();
+        int size = 16+ownerType.getBytes().length+mapName.getBytes().length+mapKey.getBytes().length;
         if(bucketNumber > 0 ){
             // ints are 4 bytes
             size += 4;

http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcbe8036/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java
b/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java
index 89f6799..07bec81 100644
--- a/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java
+++ b/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java
@@ -41,6 +41,7 @@ import com.google.inject.Inject;
 import static junit.framework.TestCase.assertNotNull;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
 
 
 @RunWith( ITRunner.class )
@@ -121,20 +122,20 @@ public class MapManagerTest {
         MapKeyResults keyResults = mm.getKeys(null, 3);
 
         assertEquals(3, keyResults.getKeys().size());
-        assertEquals(key6, keyResults.getKeys().get(0));
+        assertTrue("should contain key1", keyResults.getKeys().contains(key1));
 
         assertNotNull(keyResults.getCursor());
 
         MapKeyResults keyResults2 = mm.getKeys(keyResults.getCursor(), 3);
 
         assertEquals(3, keyResults2.getKeys().size());
-        assertEquals(key3, keyResults2.getKeys().get(0));
+        assertTrue("should contain key4", keyResults2.getKeys().contains(key4));
 
 
     }
 
     @Test
-    public void getAllKeysAfterDelete(){
+    public void testKeysOrdering(){
 
         MapManager mm = mmf.createMapManager(this.scope);
 
@@ -157,8 +158,45 @@ public class MapManagerTest {
         MapKeyResults keyResults = mm.getKeys(null, 6);
 
         assertEquals(6, keyResults.getKeys().size());
-        assertEquals(key6, keyResults.getKeys().get(0));
+        assertEquals(key1, keyResults.getKeys().get(0));
+
+
+        mm.delete(key1);
+        mm.delete(key2);
+        mm.delete(key3);
+
+        MapKeyResults keyResults2 = mm.getKeys(null, 6);
+
+        assertEquals(3, keyResults2.getKeys().size());
+        assertEquals(key4, keyResults2.getKeys().get(0));
+
+
+    }
 
+    @Test
+    public void getAllKeysAfterDelete(){
+
+        MapManager mm = mmf.createMapManager(this.scope);
+
+        final String value = "value";
+
+        final String key1 = "key1";
+        final String key2 = "key2";
+        final String key3 = "key3";
+        final String key4 = "key4";
+        final String key5 = "key5";
+        final String key6 = "key6";
+
+        mm.putString( key1, value );
+        mm.putString( key2, value );
+        mm.putString( key3, value );
+        mm.putString( key4, value );
+        mm.putString( key5, value );
+        mm.putString( key6, value );
+
+        MapKeyResults keyResults = mm.getKeys(null, 6);
+
+        assertEquals(6, keyResults.getKeys().size());
 
         mm.delete(key4);
         mm.delete(key5);
@@ -167,8 +205,6 @@ public class MapManagerTest {
         MapKeyResults keyResults2 = mm.getKeys(null, 6);
 
         assertEquals(3, keyResults2.getKeys().size());
-        assertEquals(key3, keyResults2.getKeys().get(0));
-
 
     }
 


Mime
View raw message