usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From snoopd...@apache.org
Subject [11/50] incubator-usergrid git commit: Merge branch 'two-dot-o' into USERGRID-448-appinfofix
Date Mon, 30 Mar 2015 19:58:15 GMT
Merge branch 'two-dot-o' into USERGRID-448-appinfofix

Conflicts:
	stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/7f84f4ad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/7f84f4ad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/7f84f4ad

Branch: refs/heads/two-dot-o-dev
Commit: 7f84f4ad42386564ba0bceafc1ffb5f946a9dada
Parents: 2664f6f 673c057
Author: Dave Johnson <dmjohnson@apigee.com>
Authored: Mon Mar 9 12:56:41 2015 -0400
Committer: Dave Johnson <dmjohnson@apigee.com>
Committed: Mon Mar 9 12:56:41 2015 -0400

----------------------------------------------------------------------
 .../Client/IUsergridClient.cs                   |   2 +-
 .../Usergrid.Notifications/Client/PushClient.cs |  24 +++
 .../Usergrid.Notifications/MainPage.xaml        |   1 +
 .../Usergrid.Notifications/MainPage.xaml.cs     |  29 ++-
 .../main/dist/init_instance/init_rest_server.sh |   3 -
 .../src/main/groovy/configure_usergrid.groovy   |   6 +-
 stack/awscluster/ugcluster-cf.json              |   8 +
 stack/core/pom.xml                              |   7 +-
 .../corepersistence/CpEntityManager.java        | 109 +++++++++-
 .../corepersistence/CpEntityManagerFactory.java | 149 ++++++--------
 .../corepersistence/CpRelationManager.java      |  39 +++-
 .../corepersistence/OrgApplicationCache.java    |  67 +++++++
 .../OrgApplicationCacheImpl.java                | 181 +++++++++++++++++
 .../corepersistence/results/EntityVerifier.java |   4 +-
 .../results/FilteringLoader.java                |   2 +-
 .../cassandra/CassandraPersistenceUtils.java    |   2 +-
 .../main/resources/usergrid-core-context.xml    |  12 +-
 .../cassandra/EntityManagerFactoryImplIT.java   |   1 +
 .../core/metrics/MetricsFactoryImpl.java        |   3 +-
 .../persistence/index/EntityIndexFactory.java   |   4 +-
 .../persistence/index/IndexBufferProducer.java  |  10 +-
 .../usergrid/persistence/index/IndexFig.java    |  12 +-
 .../persistence/index/guice/IndexModule.java    |   7 +-
 .../index/impl/EsEntityIndexBatchImpl.java      |  15 +-
 .../index/impl/EsEntityIndexFactoryImpl.java    |  79 ++++++++
 .../index/impl/EsEntityIndexImpl.java           | 198 +++++++++++++++----
 .../index/impl/EsIndexBufferConsumerImpl.java   |  39 +++-
 .../index/impl/EsIndexBufferProducerImpl.java   |  30 ++-
 .../persistence/index/impl/IndexingUtils.java   |   5 +-
 .../persistence/index/impl/EntityIndexTest.java |   2 +-
 .../datagenerators/EntityDataGenerator.scala    |  25 +++
 .../datagenerators/FeederGenerator.scala        |  18 +-
 .../usergrid/scenarios/EntityScenarios.scala    |   8 +-
 .../org/apache/usergrid/settings/Settings.scala |   4 +-
 .../simulations/PutCustomEntitySimulation.scala |  75 +++++++
 stack/pom.xml                                   |  24 ++-
 stack/rest/pom.xml                              |   2 +-
 .../org/apache/usergrid/rest/IndexResource.java |  19 ++
 stack/services/pom.xml                          |   7 +-
 .../services/notifications/wns/WNSAdapter.java  |  39 ++++
 .../resources/usergrid-services-context.xml     |   8 +-
 .../usergrid/management/OrganizationIT.java     |   2 +-
 42 files changed, 1056 insertions(+), 225 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7f84f4ad/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index e1a7c60,12504ed..783261a
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -18,23 -19,27 +19,12 @@@ import com.google.common.base.Optional
  import com.google.common.cache.CacheBuilder;
  import com.google.common.cache.CacheLoader;
  import com.google.common.cache.LoadingCache;
 -import com.google.inject.Guice;
  import com.google.inject.Injector;
  import com.yammer.metrics.annotation.Metered;
--import static java.lang.String.CASE_INSENSITIVE_ORDER;
--
--import java.util.*;
--import java.util.concurrent.atomic.AtomicBoolean;
--
  import org.apache.commons.lang.StringUtils;
--
  import org.apache.usergrid.corepersistence.rx.AllEntitiesInSystemObservable;
  import org.apache.usergrid.corepersistence.util.CpNamingUtils;
 -import org.apache.usergrid.persistence.AbstractEntity;
 -import org.apache.usergrid.persistence.Entity;
 -import org.apache.usergrid.persistence.EntityFactory;
 -import org.apache.usergrid.persistence.EntityManager;
 -import org.apache.usergrid.persistence.EntityManagerFactory;
 -import org.apache.usergrid.persistence.EntityRef;
 -import org.apache.usergrid.persistence.Results;
 -import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
 -import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
 +import org.apache.usergrid.persistence.*;
- 
- import static org.apache.usergrid.persistence.Schema.PROPERTY_PATH;
- import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
- import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID;
- import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
  import org.apache.usergrid.persistence.cassandra.CassandraService;
  import org.apache.usergrid.persistence.cassandra.CounterUtils;
  import org.apache.usergrid.persistence.cassandra.Setup;
@@@ -50,7 -55,7 +41,6 @@@ import org.apache.usergrid.persistence.
  import org.apache.usergrid.persistence.exceptions.ApplicationAlreadyExistsException;
  import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
  import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
--import org.apache.usergrid.persistence.exceptions.OrganizationAlreadyExistsException;
  import org.apache.usergrid.persistence.graph.Edge;
  import org.apache.usergrid.persistence.graph.GraphManager;
  import org.apache.usergrid.persistence.graph.SearchByEdgeType;
@@@ -59,8 -64,8 +49,8 @@@ import org.apache.usergrid.persistence.
  import org.apache.usergrid.persistence.index.query.Query;
  import org.apache.usergrid.persistence.model.entity.Id;
  import org.apache.usergrid.persistence.model.entity.SimpleId;
--import org.apache.usergrid.utils.UUIDUtils;
  import org.apache.usergrid.persistence.model.util.UUIDGenerator;
++import org.apache.usergrid.utils.UUIDUtils;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  import org.springframework.beans.BeansException;
@@@ -68,6 -73,6 +58,12 @@@ import org.springframework.context.Appl
  import org.springframework.context.ApplicationContextAware;
  import rx.Observable;
  
++import java.util.*;
++import java.util.concurrent.atomic.AtomicBoolean;
++
++import static java.lang.String.CASE_INSENSITIVE_ORDER;
++import static org.apache.usergrid.persistence.Schema.*;
++
  
  /**
   * Implement good-old Usergrid EntityManagerFactory with the new-fangled Core Persistence
API.
@@@ -84,11 -89,7 +80,11 @@@ public class CpEntityManagerFactory imp
      /** Have we already initialized the index for the management app? */
      private AtomicBoolean indexInitialized = new AtomicBoolean(  );
  
 +    /** Keep track of applications that already have indexes to avoid redundant re-creation.
*/
 +    private static final Set<UUID> applicationIndexesCreated = new HashSet<UUID>();
 +
 +
-     // cache of already instantiated oldAppInfo managers
+     // cache of already instantiated entity managers
      private LoadingCache<UUID, EntityManager> entityManagers
          = CacheBuilder.newBuilder().maximumSize(100).build(new CacheLoader<UUID, EntityManager>()
{
              public EntityManager load(UUID appId) { // no checked exception
@@@ -113,12 -116,9 +111,14 @@@
          this.injector = injector;
          this.managerCache = injector.getInstance( ManagerCache.class );
          this.dataMigrationManager = injector.getInstance( DataMigrationManager.class );
+         this.metricsFactory = injector.getInstance( MetricsFactory.class );
  
 +        // can be removed after everybody moves to Usergrid 2.0, default is true
 +        Properties configProps = cassandraService.getProperties();
 +        if ( configProps.getProperty("usergrid.twodoto.appinfo.migration", "true").equals("true"))
{
 +            migrateOldAppInfos();
 +        }
+         this.orgApplicationCache = new OrgApplicationCacheImpl( this );
      }
  
  
@@@ -222,8 -220,13 +220,11 @@@
      public UUID initializeApplication( String organizationName, UUID applicationId, String
name,
                                         Map<String, Object> properties ) throws Exception
{
  
 +        EntityManager em = getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID);
  
 -
 -        //Ensure our management system exists before creating our application
++        // Ensure our management system exists before creating our application
+         init();
+ 
 -        EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID);
 -
          final String appName = buildAppName( organizationName, name );
  
          // check for pre-existing application
@@@ -232,13 -234,47 +233,36 @@@
              throw new ApplicationAlreadyExistsException( appName );
          }
  
-         // create application info entity in the management app
- 
          getSetup().setupApplicationKeyspace( applicationId, appName );
+ 
 -
+         final Optional<UUID> cachedValue = orgApplicationCache.getOrganizationId(
organizationName );
+ 
 -
 -        UUID orgUuid;
 -
+         if ( !cachedValue.isPresent() ) {
+ 
+ 
+             // create new org because the specified one does not exist
+             final String orgName = organizationName;
+ 
+ 
+ 
+             try {
+                 final Entity orgInfo = em.create( "organization", new HashMap<String,
Object>() {{
+                     put( PROPERTY_NAME, orgName );
+                 }} );
 -                orgUuid = orgInfo.getUuid();
+                 //evit so it's re-loaded later
+                 orgApplicationCache.evictOrgId( name );
+             }
+             catch ( DuplicateUniquePropertyExistsException e ) {
+                 //swallow, if it exists, just get it
+                 orgApplicationCache.evictOrgId( organizationName );
 -                orgUuid = orgApplicationCache.getOrganizationId( organizationName ).get();
+             }
 -
 -        } else{
 -            orgUuid = cachedValue.get();
+         }
+ 
+         // create appinfo entry in the system app
          final UUID appId = applicationId;
 -        final UUID orgId = orgUuid;
          Map<String, Object> appInfoMap = new HashMap<String, Object>() {{
              put( PROPERTY_NAME, appName );
 -            put( "applicationUuid", appId );
 -            put( "organizationUuid", orgId );
 +            put( PROPERTY_UUID, appId );
          }};
  
          try {
@@@ -255,7 -291,9 +279,8 @@@
          }
          properties.put( PROPERTY_NAME, appName );
          EntityManager appEm = getEntityManager( applicationId );
 -
          appEm.create( applicationId, TYPE_APPLICATION, properties );
+         appEm.createIndex();
          appEm.resetRoles();
          appEm.refreshIndex();
  
@@@ -360,68 -383,10 +389,8 @@@
      }
  
  
-     @Override
-     public UUID lookupApplication( final String name ) throws Exception {
-         init();
- 
-         // attempt to look up APPLICATION_INFO by name
- 
-         UUID applicationId = null;
-         EntityManager em = getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
-         final EntityRef alias = em.getAlias( CpNamingUtils.APPLICATION_INFO, name );
-         if ( alias != null ) {
-             Entity entity = em.get(alias);
-             applicationId = (UUID) entity.getProperty("uuid");
-         }
- 
-         // below is not necessary if migrateOldAppInfos() has already run
- 
- //        if ( applicationId == null ) {
- //
- //            // maybe there is a record in the old and deprecated "appinfos" collection
- //
- //            UUID organizationId = null;
- //
- //            Query q = Query.fromQL( PROPERTY_NAME + " = '" + name + "'");
- //            Results results = em.searchCollection( em.getApplicationRef(), "appinfos"
, q);
- //            if ( !results.isEmpty() ) {
- //                Entity entity = results.iterator().next();
- //                Object uuidObject = entity.getProperty("applicationUuid");
- //                if (uuidObject instanceof UUID) {
- //                    applicationId = (UUID)uuidObject;
- //                } else {
- //                    applicationId = UUIDUtils.tryExtractUUID(uuidObject.toString());
- //                }
- //                uuidObject = entity.getProperty("organizationUuid");
- //                if (uuidObject instanceof UUID) {
- //                    organizationId = (UUID)uuidObject;
- //                } else {
- //                    organizationId = UUIDUtils.tryExtractUUID(uuidObject.toString());
- //                }
- //            }
- //
- //            if ( applicationId != null ) {
- //
- //                // copy application information into new APPLICATION_INFO collection
- //
- //                final UUID appId = applicationId;
- //                Map<String, Object> appInfoMap = new HashMap<String, Object>()
{{
- //                    put( PROPERTY_NAME, name );
- //                    put( PROPERTY_UUID, appId );
- //                }};
- //
- //                final Entity appInfo;
- //                try {
- //                    appInfo = em.create( appId, CpNamingUtils.APPLICATION_INFO, appInfoMap
);
- //                } catch (DuplicateUniquePropertyExistsException e) {
- //                    throw new ApplicationAlreadyExistsException(name);
- //                }
- //                em.createConnection( new SimpleEntityRef( Group.ENTITY_TYPE, organizationId
), "owns", appInfo );
- //                em.refreshIndex();
- //            }
- //        }
--
-         return applicationId;
 -    @Override
+     public UUID lookupApplication( String orgAppName ) throws Exception {
+         return orgApplicationCache.getApplicationId( orgAppName ).orNull();
      }
  
  

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7f84f4ad/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java
index 0000000,4baf598..23410d8
mode 000000,100644..100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java
@@@ -1,0 -1,181 +1,181 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing,
+  * software distributed under the License is distributed on an
+  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  * KIND, either express or implied.  See the License for the
+  * specific language governing permissions and limitations
+  * under the License.
+  */
+ 
+ package org.apache.usergrid.corepersistence;
+ 
+ 
+ import java.util.UUID;
+ import java.util.concurrent.ExecutionException;
+ 
+ import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+ import org.apache.usergrid.persistence.Entity;
+ import org.apache.usergrid.persistence.EntityManager;
+ import org.apache.usergrid.persistence.EntityManagerFactory;
+ import org.apache.usergrid.persistence.EntityRef;
+ import org.apache.usergrid.persistence.Results;
+ import org.apache.usergrid.persistence.index.query.Query;
+ import org.apache.usergrid.utils.UUIDUtils;
+ 
+ import com.google.common.base.Optional;
+ import com.google.common.cache.CacheBuilder;
+ import com.google.common.cache.CacheLoader;
+ import com.google.common.cache.LoadingCache;
+ 
+ import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
+ 
+ 
+ /**
+  * Implements the org app cache for faster runtime lookups.  These values are immutable,
so this LRU cache can stay
+  * full for the duration of the execution
+  */
+ public class OrgApplicationCacheImpl implements OrgApplicationCache {
+ 
+ 
+     /**
+      * Cache the pointer to our root entity manager for reference
+      */
+     private final EntityManager rootEm;
+ 
+     private final LoadingCache<String, Optional<UUID>> orgCache =
+         CacheBuilder.newBuilder().maximumSize( 10000 ).build( new CacheLoader<String,
Optional<UUID>>() {
+             @Override
+             public Optional<UUID> load( final String key ) throws Exception {
+                 return fetchOrganizationId( key );
+             }
+         } );
+ 
+ 
+     private final LoadingCache<String, Optional<UUID>> appCache =
+         CacheBuilder.newBuilder().maximumSize( 10000 ).build( new CacheLoader<String,
Optional<UUID>>() {
+             @Override
+             public Optional<UUID> load( final String key ) throws Exception {
+                 return fetchApplicationId( key );
+             }
+         } );
+ 
+ 
+     public OrgApplicationCacheImpl( final EntityManagerFactory emf ) {
 -        this.rootEm = emf.getEntityManager( CpNamingUtils.SYSTEM_APP_ID );
++        this.rootEm = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID);
+     }
+ 
+ 
+     @Override
+     public Optional<UUID> getOrganizationId( final String orgName ) {
+         try {
+             return orgCache.get( orgName );
+         }
+         catch ( ExecutionException e ) {
+             throw new RuntimeException( "Unable to load org cache", e );
+         }
+     }
+ 
+ 
+     /**
+      * Fetches the organization
+      */
+     private Optional<UUID> fetchOrganizationId( final String orgName ) {
+ 
+         try {
+             final EntityRef alias = rootEm.getAlias( "organizations", orgName );
+ 
+             if ( alias == null ) {
+                 return Optional.absent();
+             }
+ 
+             final Entity entity;
+ 
+             entity = rootEm.get( alias );
+ 
+ 
+             if ( entity == null ) {
+                 return Optional.absent();
+             }
+ 
+             return Optional.of( entity.getUuid() );
+         }
+         catch ( Exception e ) {
+             throw new RuntimeException( "Unable to load organization Id for caching", e
);
+         }
+     }
+ 
+ 
+     @Override
+     public void evictOrgId( final String orgName ) {
+         orgCache.invalidate( orgName );
+     }
+ 
+ 
+     @Override
+     public Optional<UUID> getApplicationId( final String applicationName ) {
+         try {
+             return appCache.get( applicationName );
+         }
+         catch ( ExecutionException e ) {
+             throw new RuntimeException( "Unable to load org cache", e );
+         }
+     }
+ 
+ 
+     /**
+      * Fetch our application id
+      */
+     private Optional<UUID> fetchApplicationId( final String applicationName ) {
+ 
+         try {
+             Query q = Query.fromQL( PROPERTY_NAME + " = '" + applicationName + "'" );
+ 
+ 
+             Results results = rootEm.searchCollection( rootEm.getApplicationRef(), "appinfos",
q );
+ 
+             if ( results.isEmpty() ) {
+                 return Optional.absent();
+             }
+ 
+             Entity entity = results.iterator().next();
+             Object uuidObject = entity.getProperty( "applicationUuid" );
+ 
+             final UUID value;
+             if ( uuidObject instanceof UUID ) {
+                 value = ( UUID ) uuidObject;
+             }
+             else {
+                 value = UUIDUtils.tryExtractUUID( entity.getProperty( "applicationUuid"
).toString() );
+             }
+ 
+ 
+             return Optional.of( value );
+         }
+         catch ( Exception e ) {
+             throw new RuntimeException( "Unable to retreive application id", e );
+         }
+     }
+ 
+ 
+     @Override
+     public void evictAppId( final String applicationName ) {
+         appCache.invalidate( applicationName );
+     }
+ 
+ 
+     @Override
+     public void evictAll() {
+         orgCache.invalidateAll();
+         appCache.invalidateAll();
+     }
+ }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7f84f4ad/stack/core/src/main/resources/usergrid-core-context.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7f84f4ad/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
----------------------------------------------------------------------


Mime
View raw message