usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rockers...@apache.org
Subject incubator-usergrid git commit: incremental commit for unique index
Date Mon, 09 Mar 2015 17:51:41 GMT
Repository: incubator-usergrid
Updated Branches:
  refs/heads/USERGRID-396 [created] 1bdfd108b


incremental commit for unique index


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1bdfd108
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1bdfd108
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1bdfd108

Branch: refs/heads/USERGRID-396
Commit: 1bdfd108bc5a852404da6708c643bc17ac71ce03
Parents: 710a1a3
Author: Rod Simpson <rod@rodsimpson.com>
Authored: Mon Mar 9 11:50:57 2015 -0600
Committer: Rod Simpson <rod@rodsimpson.com>
Committed: Mon Mar 9 11:50:57 2015 -0600

----------------------------------------------------------------------
 .../collection/EntityCollectionManager.java     |  9 +++
 .../persistence/collection/FieldSet.java        | 48 ++++++++++++
 .../cache/CachedEntityCollectionManager.java    | 25 ++++++-
 .../impl/EntityCollectionManagerImpl.java       | 79 +++++++++++++++++---
 .../serialization/impl/FieldSetImpl.java        | 64 ++++++++++++++++
 .../collection/EntityCollectionManagerIT.java   | 58 ++++++++++++++
 6 files changed, 269 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1bdfd108/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
index 4de18fe..6bbe912 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
@@ -54,6 +54,9 @@ public interface EntityCollectionManager {
      * Return the latest versions of the specified entityIds
      */
     public Observable<VersionSet> getLatestVersion( Collection<Id> entityId );
+    
+    
+    public Observable<FieldSet> getAllEntities(Collection<Field> fields);
 
     /**
      * Gets the Id for a field
@@ -62,6 +65,12 @@ public interface EntityCollectionManager {
     public Observable<Id> getIdField(final Field field);
 
     /**
+     * Audit a unique field, and remove any stale entries in the system
+     * @param field The field to audit within this collection scope.
+
+    public Observable<Integer> auditUniqueField(final Field field);
+     */
+    /**
      * Load all the entityIds into the observable entity set
      */
     public Observable<EntitySet> load(Collection<Id> entityIds);

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1bdfd108/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/FieldSet.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/FieldSet.java
b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/FieldSet.java
new file mode 100644
index 0000000..c46fa3b
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/FieldSet.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.persistence.collection;
+
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.field.Field;
+
+
+/**
+ * Represents a set of entities
+ */
+public interface FieldSet {
+
+    /**
+     * Get the entity from the result set
+     * @param field, Return the entity with the field
+     * @return
+     */
+    public MvccEntity getEntity(Field<?> field);
+
+    /**
+     * Get the number of entities in this set
+     * @return
+     */
+    public int size();
+
+    /**
+     * Return true if the set is empty
+     * @return
+     */
+    public boolean isEmpty();
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1bdfd108/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/cache/CachedEntityCollectionManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/cache/CachedEntityCollectionManager.java
b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/cache/CachedEntityCollectionManager.java
index 186aafa..a111f8d 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/cache/CachedEntityCollectionManager.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/cache/CachedEntityCollectionManager.java
@@ -20,13 +20,19 @@
 package org.apache.usergrid.persistence.collection.cache;
 
 
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
+import java.util.UUID;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntitySet;
-import org.apache.usergrid.persistence.collection.VersionSet;
+import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import org.apache.usergrid.persistence.collection.*;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
+import org.apache.usergrid.persistence.collection.serialization.impl.FieldSetImpl;
 import org.apache.usergrid.persistence.core.util.Health;
 import org.apache.usergrid.persistence.model.entity.Entity;
 import org.apache.usergrid.persistence.model.entity.Id;
@@ -39,8 +45,10 @@ import com.google.common.cache.LoadingCache;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
 import rx.Observable;
 import rx.functions.Action1;
+import rx.functions.Func1;
 
 
 @Singleton
@@ -74,6 +82,17 @@ public class CachedEntityCollectionManager implements EntityCollectionManager
{
                                   .build();
     }
 
+    public Observable<FieldSet> getAllEntities(final Collection<Field> fields)
{
+        return rx.Observable.just(fields).map( new Func1<Collection<Field>, FieldSet>()
{
+            @Override
+            public FieldSet call( Collection<Field> fields ) {
+
+                    final FieldSet response = new FieldSetImpl(fields.size());
+
+                    return response;
+            }
+        } );
+    }
 
     @Override
     public Observable<Entity> write( final Entity entity ) {

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1bdfd108/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 7c467c6..5505e3f 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -19,18 +19,15 @@
 package org.apache.usergrid.persistence.collection.impl;
 
 
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
+import java.util.*;
 
+import com.netflix.astyanax.MutationBatch;
+import org.apache.usergrid.persistence.collection.*;
+import org.apache.usergrid.persistence.collection.serialization.impl.FieldSetImpl;
+import org.apache.usergrid.persistence.core.rx.ObservableIterator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntitySet;
-import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.collection.VersionSet;
 import org.apache.usergrid.persistence.collection.guice.Write;
 import org.apache.usergrid.persistence.collection.guice.WriteUpdate;
 import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
@@ -65,9 +62,6 @@ import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
 import com.netflix.astyanax.model.ColumnFamily;
 import com.netflix.astyanax.model.CqlResult;
 import com.netflix.astyanax.serializers.StringSerializer;
-import org.apache.usergrid.persistence.collection.EntityDeletedFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
 import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
 import org.apache.usergrid.persistence.core.task.Task;
 import org.apache.usergrid.persistence.core.task.TaskExecutor;
@@ -293,6 +287,69 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager
{
         } );
     }
 
+    @Override
+    public Observable<FieldSet> getAllEntities(final Collection<Field> fields)
{
+        return rx.Observable.just(fields).map( new Func1<Collection<Field>, FieldSet>()
{
+            @Override
+            public FieldSet call( Collection<Field> fields ) {
+                try {
+
+                    final UUID startTime = UUIDGenerator.newTimeUUID();
+
+                    UniqueValueSet set = uniqueValueSerializationStrategy.load( collectionScope,
fields );
+
+
+                    //loop through each field, and construct an entity load
+                    List<Id> entityIds = new ArrayList<>(fields.size());
+                    List<UniqueValue> uniqueValues = new ArrayList<>(fields.size());
+
+                    for(final Field expectedField: fields) {
+
+                        UniqueValue value = set.getValue(expectedField.getName());
+
+                        entityIds.add(value.getEntityId());
+                        uniqueValues.add(value);
+                    }
+
+                    final EntitySet entitySet = entitySerializationStrategy.load(collectionScope,
entityIds, startTime);
+
+                    //now loop through and ensure the entities are there.
+                    final MutationBatch deleteBatch = keyspace.prepareMutationBatch();
+
+                    final FieldSetImpl response = new FieldSetImpl(fields.size());
+
+                    for(final UniqueValue expectedUnique: uniqueValues) {
+                        final MvccEntity entity = entitySet.getEntity(expectedUnique.getEntityId());
+
+                        //bad unique value, delete this, it's inconsistent
+                        if(entity == null || !entity.getEntity().isPresent()){
+                            final MutationBatch valueDelete = uniqueValueSerializationStrategy.delete(collectionScope,
expectedUnique);
+                            deleteBatch.mergeShallow(valueDelete);
+                            continue;
+                        }
+
+
+                        //else add it to our result set
+                        response.addEntity(expectedUnique.getField(),entity);
+
+                    }
+
+                    //fire and forget, we don't care.  We'll repair it again if we have to
+                    deleteBatch.executeAsync();
+
+                    return response;
+
+
+                }
+                catch ( ConnectionException e ) {
+                    logger.error( "Failed to getIdField", e );
+                    throw new RuntimeException( e );
+                }
+            }
+        } );
+    }
+
+   
 
     @Override
     public Observable<Entity> update( final Entity entity ) {

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1bdfd108/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/FieldSetImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/FieldSetImpl.java
b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/FieldSetImpl.java
new file mode 100644
index 0000000..241d5c2
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/FieldSetImpl.java
@@ -0,0 +1,64 @@
+package org.apache.usergrid.persistence.collection.serialization.impl;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import org.apache.usergrid.persistence.collection.FieldSet;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.model.field.Field;
+
+import java.util.HashMap;
+import java.util.Map;
+
+
+public class FieldSetImpl implements FieldSet {
+
+
+    private final Map<Field<?>, MvccEntity> entities;
+
+
+    public FieldSetImpl(
+            final int expectedSize) {
+        this.entities = new HashMap<>( expectedSize );
+    }
+
+
+    public void addEntity(final Field<?> field,  final MvccEntity entity ) {
+        entities.put( field, entity );
+    }
+
+
+    @Override
+    public MvccEntity getEntity( final Field<?> field) {
+        return entities.get( field );
+    }
+
+
+
+
+    @Override
+    public int size() {
+        return entities.size();
+    }
+
+
+    @Override
+    public boolean isEmpty() {
+        return entities.size() == 0;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1bdfd108/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
index 5eba979..f33b3d0 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
@@ -23,6 +23,9 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.UUID;
 
+import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+import org.apache.usergrid.persistence.core.guice.ProxyImpl;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -75,6 +78,14 @@ public class EntityCollectionManagerIT {
     private SerializationFig serializationFig;
 
 
+    @Inject
+    private UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
+
+    @Inject
+    @ProxyImpl
+    private MvccEntitySerializationStrategy entitySerializationStrategy;
+
+
     @Test
     public void write() {
 
@@ -743,4 +754,51 @@ public class EntityCollectionManagerIT {
         //override our default
         SetConfigTestBypass.setValueByPass( serializationFig, "getMaxEntitySize", currentMaxSize
+ "" );
     }
+    
+    @Test
+    public void invalidNameRepair() {
+        
+        //write an entity with a unique field
+        CollectionScope context = 
+                new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test"
), "test" );
+
+        Entity newEntity = new Entity( new SimpleId( "test" ) );
+        
+        newEntity.setField( new IntegerField( "count", 5, true ) );
+        newEntity.setField( new StringField( "yes", "fred", true ) );
+
+        EntityCollectionManager manager = factory.createCollectionManager( context );
+
+        Observable<Entity> observable = manager.write( newEntity );
+
+        Entity createReturned = observable.toBlocking().lastOrDefault( null );
+
+
+        assertNotNull( "Id was assigned", createReturned.getId() );
+        assertNotNull( "Version was assigned", createReturned.getVersion() );
+
+
+        Observable<Entity> loadObservable = manager.load( createReturned.getId() );
+
+        Entity loadReturned = loadObservable.toBlocking().lastOrDefault( null );
+
+        assertEquals( "Same value", createReturned, loadReturned );
+        //load an entity by it's unique field
+        
+        //verify the entity is correct.
+        
+        //use the entity serializationStrategy to remove the entity data.
+        
+        //try to load via the unique field
+        
+        //verify no entity returned
+        
+        //user the unique serialization to verify it's been deleted from cassandra
+        
+
+        
+                
+                
+                
+    }
 }


Mime
View raw message