usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From g...@apache.org
Subject [24/35] usergrid git commit: Merge branch 'release'
Date Mon, 25 Jan 2016 21:18:16 GMT
Merge branch 'release'


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c4813b5f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c4813b5f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c4813b5f

Branch: refs/heads/USERGRID-1047
Commit: c4813b5f5ec0b9b56f53a582ddc03a19801ddb3c
Parents: d796b68 62ad5a8
Author: Michael Russo <michaelarusso@gmail.com>
Authored: Thu Jan 21 16:13:03 2016 -0800
Committer: Michael Russo <michaelarusso@gmail.com>
Committed: Thu Jan 21 16:24:41 2016 -0800

----------------------------------------------------------------------
 .../main/resources/usergrid-default.properties  |  10 +-
 .../corepersistence/CpEntityManagerFactory.java |  18 +-
 .../corepersistence/CpRelationManager.java      |  54 ++--
 .../usergrid/corepersistence/CpSetup.java       |  14 +-
 .../read/traverse/AbstractReadGraphFilter.java  |  37 ++-
 .../persistence/EntityManagerFactory.java       |   8 +-
 .../usergrid/persistence/EntityManagerIT.java   |  33 +++
 .../graph/impl/stage/EdgeDeleteRepairImpl.java  |   8 +-
 .../model/entity/MapToEntityConverter.java      |  18 +-
 .../persistence/model/field/FieldTypeName.java  |   3 +-
 .../persistence/model/field/NullField.java      |  44 +++
 .../model/entity/MapToEntityConverterTest.java  |  28 ++
 .../persistence/model/field/EntityTest.java     |   9 +-
 .../persistence/index/impl/EntityField.java     |  11 +
 .../index/impl/EntityMappingParser.java         |  16 +-
 .../index/impl/EntityToMapConverter.java        |   3 +-
 .../persistence/index/impl/FieldParser.java     |   5 +-
 .../persistence/index/impl/IndexingUtils.java   |   1 +
 .../persistence/index/impl/EntityIndexTest.java |  69 ++++-
 .../index/impl/EntityToMapConverterTest.java    | 149 +++++++---
 .../index/impl/IndexLoadTestsIT.java            |   1 +
 .../management/AppInfoMigrationPlugin.java      |   3 +-
 .../usergrid/management/ManagementService.java  |   4 +-
 .../cassandra/ManagementServiceImpl.java        |  25 +-
 .../migration/AppInfoMigrationPluginTest.java   |   2 +-
 .../apache/usergrid/tools/DupAdminRepair.java   | 297 -------------------
 .../org/apache/usergrid/tools/DupOrgRepair.java | 263 ----------------
 .../org/apache/usergrid/tools/ExportApp.java    |  16 +-
 .../org/apache/usergrid/tools/ImportAdmins.java |  81 +----
 .../apache/usergrid/tools/PopulateSample.java   |   2 +-
 .../org/apache/usergrid/tools/ToolBase.java     |   6 +-
 .../apache/usergrid/tools/bean/MetricQuery.java |   7 +-
 .../apache/usergrid/tools/ExportAppTest.java    |   6 +-
 .../usergrid/tools/ExportImportAdminsTest.java  |   3 +-
 34 files changed, 485 insertions(+), 769 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 0e8a6df,8b6feed..e3a0933
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -225,12 -225,10 +224,12 @@@ public class CpEntityManagerFactory imp
  
          applicationId = applicationId==null ?  UUIDGenerator.newTimeUUID() : applicationId;
  
 -        logger.debug( "New application orgName {} orgAppName {} id {} ",
 -            new Object[] { orgName, name, applicationId.toString() } );
 +        if (logger.isDebugEnabled()) {
 +            logger.debug("New application orgName {} orgAppName {} id {} ",
 +                new Object[]{orgName, name, applicationId.toString()});
 +        }
  
-         return initializeApplicationV2( orgName, applicationId, appName, properties );
+         return initializeApplicationV2( orgName, applicationId, appName, properties, forMigration);
      }
  
  

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 68f22d8,415fb5e..fa15f8f
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@@ -698,32 -698,10 +700,11 @@@ public class CpRelationManager implemen
          indexService.queueNewEdge( applicationScope, targetEntity, edge );
  
  
-         //now read all older versions of an edge, and remove them.  Finally calling delete
-         final SearchByEdge searchByEdge =
-             new SimpleSearchByEdge( edge.getSourceNode(), edge.getType(), edge.getTargetNode(),
Long.MAX_VALUE,
-                 SearchByEdgeType.Order.DESCENDING, Optional.absent() );
- 
- 
-         //load our versions, only retain the most recent one
-         gm.loadEdgeVersions(searchByEdge).skip(1).flatMap(edgeToDelete -> {
- 
-             if (logger.isDebugEnabled()) {
-                 logger.debug("Marking edge {} for deletion", edgeToDelete);
-             }
+         // remove any duplicate edges (keeps the duplicate edge with same timestamp)
+         removeDuplicateEdgesAsync(gm, edge);
  
-             return gm.markEdge(edgeToDelete );
-         }).lastOrDefault(null).doOnNext(lastEdge -> {
-             //no op if we hit our default
-             if (lastEdge == null) {
-                 return;
-             }
- 
-             //don't queue delete b/c that de-indexes, we need to delete the edges only since
we have a version still existing to index.
- 
-             gm.deleteEdge(lastEdge).toBlocking().lastOrDefault(null); // this should throw
an exception
-         }).toBlocking().lastOrDefault(null);//this should throw an exception
  
 +
          return connection;
      }
  

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
index f62fee1,dd59b77..d7d6294
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
@@@ -175,12 -175,11 +175,13 @@@ public abstract class AbstractReadGraph
                  return !isDeleted && !isSourceNodeDeleted && !isTargetNodeDelete;
  
  
-             })
+             })  // any non-deleted edges should be de-duped here so the results are unique
+                 .distinct( new EdgeDistinctKey() )
                  //set the edge state for cursors
                  .doOnNext( edge -> {
 -                    logger.trace( "Seeking over edge {}", edge );
 +                    if (logger.isTraceEnabled()) {
 +                        logger.trace("Seeking over edge {}", edge);
 +                    }
                      edgeCursorState.update( edge );
                  } )
  

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index cb44d4d,2bd1ccb..751adc8
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@@ -23,10 -22,8 +22,8 @@@ import java.util.UUID
  
  import com.google.common.base.Optional;
  
- import org.apache.usergrid.corepersistence.index.ReIndexService;
  import org.apache.usergrid.persistence.core.util.Health;
 -import org.apache.usergrid.persistence.index.IndexRefreshCommand;
 +import org.apache.usergrid.persistence.index.EntityIndex;
- import rx.Observable;
  
  
  /**

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
----------------------------------------------------------------------
diff --cc stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
index 0f257fb,bc693ce..cb3a728
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
@@@ -584,4 -583,37 +584,37 @@@ public class EntityManagerIT extends Ab
  
          assertNotNull( em.get( user.getUuid() ) );
      }
+ 
+ 
+     @Test
+     public void testFilteringOfDuplicateEdges() throws Exception {
 -        LOG.info( "EntityManagerIT.testFilteringOfDuplicateEdges" );
++        logger.info( "EntityManagerIT.testFilteringOfDuplicateEdges" );
+ 
+         EntityManager em = app.getEntityManager();
+ 
+         Map<String, Object> properties = new LinkedHashMap<String, Object>();
+         properties.put( "name", "fluffy1" );
+ 
+         Entity entity = em.create( "fluffy", properties );
+ 
+ 
+         EntityRef appRef =  new SimpleEntityRef("application", app.getId());
+         EntityRef entityRef = new SimpleEntityRef(entity.getType(), entity.getUuid());
+ 
+         assertNotNull( entity );
+ 
+ 
+         // create duplicate edges
+         em.addToCollection(appRef, "fluffies", entityRef);
+         em.addToCollection(appRef, "fluffies", entityRef);
+ 
+         //app.refreshIndex();
+ 
+         Results results = em.getCollection(appRef,
+             "fluffies", null, 10, Level.ALL_PROPERTIES, true);
 -        
++
+         // we should be filtering duplicate edges so only assert 1 result back and not the
# of edges
+         assertEquals(1, results.getEntities().size());
+     }
+ 
  }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/FieldTypeName.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/FieldTypeName.java
index 8b0cdc3,975c6bb..73f872d
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/FieldTypeName.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/FieldTypeName.java
@@@ -40,5 -40,5 +40,6 @@@ public enum FieldTypeName 
      SET,
      STRING,
      UUID,
-     DISTANCE
++    DISTANCE,
+     NULL
  }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/corepersistence/model/src/test/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverterTest.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/model/src/test/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverterTest.java
index a5aa66b,03dfa11..23e6696
--- a/stack/corepersistence/model/src/test/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverterTest.java
+++ b/stack/corepersistence/model/src/test/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverterTest.java
@@@ -70,7 -70,8 +70,8 @@@ public class MapToEntityConverterTest 
          data.put("floatField", floatField);
          data.put("location", coordinates);
          data.put("objectField", objectField);
 -        data.put("nullField", null)
 -;
++        data.put("nullField", null);
 +
  
          // convert the map to an entity
          MapToEntityConverter converter = new MapToEntityConverter();
@@@ -112,4 -114,29 +114,30 @@@
          assertTrue(entity.getField("parentArray").getTypeName() == FieldTypeName.LIST);
      }
  
++
+     @Test
+     public void testNullWithinArrays() {
+ 
+         // build top-level map
+         final Map<String,Object> data = new HashMap<>(1);
+ 
+         final List<Object> arrayNullValues = new ArrayList<>(1);
+         arrayNullValues.add(null);
+         arrayNullValues.add(null);
+ 
+         // add the nested list to the map
+         data.put("arrayNullValues", arrayNullValues);
+ 
+         // convert the map to an entity
+         MapToEntityConverter converter = new MapToEntityConverter();
+         Entity entity = converter.fromMap(data, true);
+ 
+         // make sure the nested array got converted into a ListField
+         assertTrue(entity.getField("arrayNullValues").getTypeName() == FieldTypeName.LIST);
+ 
+         List arrayReturned = (List) entity.getField("arrayNullValues").getValue();
+ 
+         assertTrue( arrayReturned.get(0) == null);
+     }
+ 
  }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/services/src/main/java/org/apache/usergrid/management/AppInfoMigrationPlugin.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/services/src/test/java/org/apache/usergrid/corepersistence/migration/AppInfoMigrationPluginTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java
----------------------------------------------------------------------
diff --cc stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java
index 9e522d2,a90a83b..9a9c522
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java
@@@ -142,7 -144,7 +144,8 @@@ public class ExportApp extends Exportin
              return Observable.create( new EntityObservable( em, collection ) )
                      .doOnNext( new EntityWriteAction() ).subscribeOn( writeScheduler );
  
-         }, writeThreadCount ).flatMap( exportEntity -> {
++
+         } ).flatMap( exportEntity -> {
  
              return Observable.create( new ConnectionsObservable( em, exportEntity ) )
                      .doOnNext( new ConnectionWriteAction() ).subscribeOn( writeScheduler
);

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
----------------------------------------------------------------------
diff --cc stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
index c8c4d99,c1a60b8..0c6337f
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
@@@ -50,8 -50,6 +50,9 @@@ import java.util.concurrent.atomic.Atom
  import static org.apache.usergrid.persistence.Schema.PROPERTY_TYPE;
  import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID;
  
 +import org.apache.usergrid.persistence.index.query.Identifier;
 +
++
  /**
   * Import Admin Users and metadata including organizations and passwords.
   *

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java
----------------------------------------------------------------------
diff --cc stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java
index 48e2af6,48e2af6..a9077be
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java
@@@ -66,7 -66,7 +66,7 @@@ public class PopulateSample extends Too
          logger.info( "Starting test..." );
          startSpring();
  
--        UserInfo user = managementService.createAdminUser( "admin", "admin", "admin@ug.com",
"none", false, false );
++        UserInfo user = managementService.createAdminUser( null, "admin", "admin", "admin@ug.com",
"none", false, false );
  
          logger.info( "Creating organization: sample-organization" );
          // management

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java
----------------------------------------------------------------------
diff --cc stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java
index 8402dd2,49b110b..85582a3
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java
@@@ -21,12 -25,7 +25,9 @@@ import java.util.ArrayList
  import java.util.List;
  import java.util.UUID;
  
- import org.apache.usergrid.persistence.*;
  
- import com.google.common.base.Preconditions;
- import org.apache.usergrid.persistence.index.query.CounterResolution;
 +
 +
  /** @author zznate */
  public class MetricQuery {
  

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
----------------------------------------------------------------------
diff --cc stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
index 429e700,2946ddb..cbc06c2
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
@@@ -17,9 -17,7 +17,10 @@@
  package org.apache.usergrid.tools;
  
  import org.apache.commons.lang.RandomStringUtils;
 +import org.apache.usergrid.ServiceITSetup;
 +import org.apache.usergrid.ServiceITSetupImpl;
 +import org.junit.ClassRule;
+ import org.apache.usergrid.services.AbstractServiceIT;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  
@@@ -41,9 -38,7 +42,10 @@@ public class ExportAppTest extends Abst
      int NUM_ENTITIES = 50;
      int NUM_CONNECTIONS = 3;
  
 +    @ClassRule
 +    public static ServiceITSetup setup = new ServiceITSetupImpl();
 +
+ 
      @org.junit.Test
      public void testBasicOperation() throws Exception {
  
@@@ -53,16 -48,12 +55,18 @@@
  
          String orgName = "org_" + rand;
          String appName = "app_" + rand;
 +        String userName = "user_" + rand;
 +
+ 
          ExportDataCreator creator = new ExportDataCreator();
          creator.startTool( new String[] {
                  "-organization", orgName,
                  "-application", appName,
 -            "-host", "localhost:" + setup.getCassSvc().getCassandraHostConfigurator().getPort()
 +                "-username", userName,
 +                "-host", "localhost:9160",
 +                "-eshost", "localhost:9200",
 +                "-escluster", "elasticsearch"
++
          }, false);
  
          long start = System.currentTimeMillis();

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java
----------------------------------------------------------------------
diff --cc stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java
index 745ec9a,ceb6074..f0d599c
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java
@@@ -48,16 -46,9 +49,16 @@@ import static junit.framework.TestCase.
  import static org.junit.Assert.*;
  
  
- public class ExportImportAdminsTest {
+ public class ExportImportAdminsTest extends AbstractServiceIT {
      static final Logger logger = LoggerFactory.getLogger( ExportImportAdminsTest.class );
  
 +    @ClassRule
 +    public static ServiceITSetup setup = new ServiceITSetupImpl();
 +
 +    public ExportImportAdminsTest() {
 +        //logger.info("\n\n\nCREATING TEST -------------------------------------------\n\n\n");
 +    }
 +
      @org.junit.Test
      public void testExportUserAndOrg() throws Exception {
  


Mime
View raw message