usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From g...@apache.org
Subject [15/38] incubator-usergrid git commit: Added caching to org/app lookup
Date Thu, 12 Mar 2015 22:41:22 GMT
Added caching to org/app lookup


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4c8727c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4c8727c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4c8727c1

Branch: refs/heads/USERGRID-396
Commit: 4c8727c1dc0e5f02a498849be1a19e6342f6accf
Parents: 2c54a2b
Author: Todd Nine <tnine@apigee.com>
Authored: Fri Mar 6 15:13:49 2015 -0700
Committer: Todd Nine <tnine@apigee.com>
Committed: Fri Mar 6 15:13:49 2015 -0700

----------------------------------------------------------------------
 .../corepersistence/CpEntityManager.java        |   1 +
 .../corepersistence/CpEntityManagerFactory.java | 116 ++++--------
 .../corepersistence/OrgApplicationCache.java    |  67 +++++++
 .../OrgApplicationCacheImpl.java                | 181 +++++++++++++++++++
 .../main/resources/usergrid-core-context.xml    |  14 +-
 .../persistence/index/EntityIndexFactory.java   |   4 +-
 .../resources/usergrid-services-context.xml     |   8 +-
 7 files changed, 296 insertions(+), 95 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 5619382..652e084 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -676,6 +676,7 @@ public class CpEntityManager implements EntityManager {
 
     @Override
     public RelationManager getRelationManager( EntityRef entityRef ) {
+        Preconditions.checkNotNull( entityRef, "entityRef cannot be null" );
         CpRelationManager rmi = new CpRelationManager();
         rmi.init( this, emf, applicationId, entityRef, null );
         return rmi;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 9aa36a0..16de53e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -15,6 +15,7 @@
  */
 package org.apache.usergrid.corepersistence;
 
+import com.google.common.base.Optional;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
@@ -98,6 +99,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
             }
         });
 
+    private final OrgApplicationCache orgApplicationCache;
+
 
     private ManagerCache managerCache;
     private DataMigrationManager dataMigrationManager;
@@ -116,7 +119,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
         this.managerCache = injector.getInstance( ManagerCache.class );
         this.dataMigrationManager = injector.getInstance( DataMigrationManager.class );
 
-
+        this.orgApplicationCache = new OrgApplicationCacheImpl( this );
     }
 
 
@@ -156,18 +159,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory,
Application
     }
 
 
-//    public ManagerCache getManagerCache() {
-//
-//        if ( managerCache == null ) {
-//            managerCache = injector.getInstance( ManagerCache.class );
-//
-//            dataMigrationManager = injector.getInstance( DataMigrationManager.class );
-//        }
-//        return managerCache;
-//    }
-
-
-
     @Override
     public EntityManager getEntityManager(UUID applicationId) {
         try {
@@ -207,15 +198,16 @@ public class CpEntityManagerFactory implements EntityManagerFactory,
Application
 
         String appName = buildAppName( orgName, name );
 
-        UUID applicationId = lookupApplication( appName );
 
-        if ( applicationId != null ) {
+        final Optional<UUID> appId = orgApplicationCache.getApplicationId( appName
);
+
+        if ( appId.isPresent() ) {
             throw new ApplicationAlreadyExistsException( name );
         }
 
-        applicationId = UUIDGenerator.newTimeUUID();
+        UUID applicationId = UUIDGenerator.newTimeUUID();
 
-        logger.debug( "New application orgName {} name {} id {} ",
+        logger.debug( "New application orgName {} orgAppName {} id {} ",
                 new Object[] { orgName, name, applicationId.toString() } );
 
         initializeApplication( orgName, applicationId, appName, properties );
@@ -233,6 +225,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory,
Application
                                        Map<String, Object> properties ) throws Exception
{
 
 
+
+        //Ensure our management system exists before creating our application
+        init();
+
         EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID);
 
         final String appName = buildAppName( organizationName, name );
@@ -244,8 +240,14 @@ public class CpEntityManagerFactory implements EntityManagerFactory,
Application
 
         getSetup().setupApplicationKeyspace( applicationId, appName );
 
-        UUID orgUuid = lookupOrganization( organizationName );
-        if ( orgUuid == null ) {
+
+        final Optional<UUID> cachedValue = orgApplicationCache.getOrganizationId( name
);
+
+
+        final UUID orgUuid;
+
+        if ( !cachedValue.isPresent() ) {
+
 
             // create new org because the specified one does not exist
             final String orgName = organizationName;
@@ -263,6 +265,11 @@ public class CpEntityManagerFactory implements EntityManagerFactory,
Application
 
             em.refreshIndex();
             orgUuid = orgInfo.getUuid();
+
+            //evit so it's re-loaded later
+            orgApplicationCache.evictOrgId( name );
+        } else{
+            orgUuid = cachedValue.get();
         }
 
         // create appinfo entry in the system app
@@ -293,6 +300,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory,
Application
         appEm.refreshIndex();
 
         logger.info("Initialized application {}", appName );
+
+        //evict app Id from cache
+        orgApplicationCache.evictAppId( appName );
+
         return applicationId;
     }
 
@@ -374,71 +385,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory,
Application
     }
 
 
-    public UUID lookupOrganization( String name ) throws Exception {
-        init();
-
-
-        //        Query q = Query.fromQL(PROPERTY_NAME + " = '" + name + "'");
-        EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID );
-
-
-        final EntityRef alias = em.getAlias( "organizations", name );
-
-        if ( alias == null ) {
-            return null;
-        }
-
-        final Entity entity = em.get( alias );
-
-        if ( entity == null ) {
-            return null;
-        }
-
-        return entity.getUuid();
-        //        Results results = em.searchCollection( em.getApplicationRef(), "organizations",
q );
-        //
-        //        if ( results.isEmpty() ) {
-        //            return null;
-        //        }
-        //
-        //        return results.iterator().next().getUuid();
-    }
-
 
     @Override
-    public UUID lookupApplication( String name ) throws Exception {
-        init();
-
-        // TODO: why does this not work for restored apps
-
-//        EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID );
-//        final EntityRef alias = em.getAlias( CpNamingUtils.APPINFOS, name );
-//        if ( alias == null ) {
-//            return null;
-//        }
-//        final Entity entity = em.get( alias );
-//        if ( entity == null ) {
-//            return null;
-//        }
-//        final UUID property = ( UUID ) entity.getProperty( "applicationUuid" );
-//        return property;
-
-        Query q = Query.fromQL( PROPERTY_NAME + " = '" + name + "'");
-
-        EntityManager em = getEntityManager(CpNamingUtils.SYSTEM_APP_ID);
-
-        Results results = em.searchCollection( em.getApplicationRef(), "appinfos", q);
-
-        if ( results.isEmpty() ) {
-            return null;
-        }
-
-        Entity entity = results.iterator().next();
-        Object uuidObject = entity.getProperty("applicationUuid");
-        if ( uuidObject instanceof UUID ) {
-            return (UUID)uuidObject;
-        }
-        return UUIDUtils.tryExtractUUID( entity.getProperty("applicationUuid").toString()
);
+    public UUID lookupApplication( String orgAppName ) throws Exception {
+        return orgApplicationCache.getApplicationId( orgAppName ).orNull();
     }
 
 
@@ -632,7 +582,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
             em.update( propsEntity );
 
         } catch (Exception ex) {
-            logger.error("Error deleting service property name: " + name, ex);
+            logger.error("Error deleting service property orgAppName: " + name, ex);
             return false;
         }
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCache.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCache.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCache.java
new file mode 100644
index 0000000..b20dbe1
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCache.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.Entity;
+
+import com.google.common.base.Optional;
+
+
+/**
+ * A simple cache interface for looking up entities from an EM
+ */
+public interface OrgApplicationCache {
+
+
+    /**
+     * Get an entity by it's alias property.  The result is cached. To clear it call evict
or evict all
+     * @param
+     * @return
+     */
+    public Optional<UUID> getOrganizationId(final String orgName);
+
+    /**
+     * Evict the org by name
+     * @param orgName
+     */
+    public void evictOrgId(final String orgName);
+
+    /**
+     * Evict the application by name
+     * @param applicationName
+     * @return
+     */
+    public Optional<UUID> getApplicationId(final String applicationName);
+
+
+    /**
+     * Evict the app id by the name
+     */
+    public void evictAppId(final String applicationname);
+
+
+    /**
+     * Evict all caches
+     */
+    public void evictAll();
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java
new file mode 100644
index 0000000..4baf598
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence;
+
+
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.utils.UUIDUtils;
+
+import com.google.common.base.Optional;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
+import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
+
+
+/**
+ * Implements the org app cache for faster runtime lookups.  These values are immutable,
so this LRU cache can stay
+ * full for the duration of the execution
+ */
+public class OrgApplicationCacheImpl implements OrgApplicationCache {
+
+
+    /**
+     * Cache the pointer to our root entity manager for reference
+     */
+    private final EntityManager rootEm;
+
+    private final LoadingCache<String, Optional<UUID>> orgCache =
+        CacheBuilder.newBuilder().maximumSize( 10000 ).build( new CacheLoader<String,
Optional<UUID>>() {
+            @Override
+            public Optional<UUID> load( final String key ) throws Exception {
+                return fetchOrganizationId( key );
+            }
+        } );
+
+
+    private final LoadingCache<String, Optional<UUID>> appCache =
+        CacheBuilder.newBuilder().maximumSize( 10000 ).build( new CacheLoader<String,
Optional<UUID>>() {
+            @Override
+            public Optional<UUID> load( final String key ) throws Exception {
+                return fetchApplicationId( key );
+            }
+        } );
+
+
+    public OrgApplicationCacheImpl( final EntityManagerFactory emf ) {
+        this.rootEm = emf.getEntityManager( CpNamingUtils.SYSTEM_APP_ID );
+    }
+
+
+    @Override
+    public Optional<UUID> getOrganizationId( final String orgName ) {
+        try {
+            return orgCache.get( orgName );
+        }
+        catch ( ExecutionException e ) {
+            throw new RuntimeException( "Unable to load org cache", e );
+        }
+    }
+
+
+    /**
+     * Fetches the organization
+     */
+    private Optional<UUID> fetchOrganizationId( final String orgName ) {
+
+        try {
+            final EntityRef alias = rootEm.getAlias( "organizations", orgName );
+
+            if ( alias == null ) {
+                return Optional.absent();
+            }
+
+            final Entity entity;
+
+            entity = rootEm.get( alias );
+
+
+            if ( entity == null ) {
+                return Optional.absent();
+            }
+
+            return Optional.of( entity.getUuid() );
+        }
+        catch ( Exception e ) {
+            throw new RuntimeException( "Unable to load organization Id for caching", e );
+        }
+    }
+
+
+    @Override
+    public void evictOrgId( final String orgName ) {
+        orgCache.invalidate( orgName );
+    }
+
+
+    @Override
+    public Optional<UUID> getApplicationId( final String applicationName ) {
+        try {
+            return appCache.get( applicationName );
+        }
+        catch ( ExecutionException e ) {
+            throw new RuntimeException( "Unable to load org cache", e );
+        }
+    }
+
+
+    /**
+     * Fetch our application id
+     */
+    private Optional<UUID> fetchApplicationId( final String applicationName ) {
+
+        try {
+            Query q = Query.fromQL( PROPERTY_NAME + " = '" + applicationName + "'" );
+
+
+            Results results = rootEm.searchCollection( rootEm.getApplicationRef(), "appinfos",
q );
+
+            if ( results.isEmpty() ) {
+                return Optional.absent();
+            }
+
+            Entity entity = results.iterator().next();
+            Object uuidObject = entity.getProperty( "applicationUuid" );
+
+            final UUID value;
+            if ( uuidObject instanceof UUID ) {
+                value = ( UUID ) uuidObject;
+            }
+            else {
+                value = UUIDUtils.tryExtractUUID( entity.getProperty( "applicationUuid" ).toString()
);
+            }
+
+
+            return Optional.of( value );
+        }
+        catch ( Exception e ) {
+            throw new RuntimeException( "Unable to retreive application id", e );
+        }
+    }
+
+
+    @Override
+    public void evictAppId( final String applicationName ) {
+        appCache.invalidate( applicationName );
+    }
+
+
+    @Override
+    public void evictAll() {
+        orgCache.invalidateAll();
+        appCache.invalidateAll();
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/resources/usergrid-core-context.xml
----------------------------------------------------------------------
diff --git a/stack/core/src/main/resources/usergrid-core-context.xml b/stack/core/src/main/resources/usergrid-core-context.xml
index c8ce0d8..cd40d6d 100644
--- a/stack/core/src/main/resources/usergrid-core-context.xml
+++ b/stack/core/src/main/resources/usergrid-core-context.xml
@@ -54,7 +54,7 @@
 		<constructor-arg value="${cassandra.url}" />
         <!-- set the pool size if it's available.  If not go with 50 -->
         <property name="maxActive" value="${cassandra.connections:50}"/>
-        <!--<property name="clockResolution" ref="microsecondsTimeResolution" />-->
+        <!--<property orgAppName="clockResolution" ref="microsecondsTimeResolution"
/>-->
         <property name="opTimer" ref="taggedOpTimer"/>
         <property name="loadBalancingPolicy" ref="loadBalancingPolicy"/>
 	</bean>
@@ -70,7 +70,7 @@
     <bean id="loadBalancingPolicy" class="me.prettyprint.cassandra.connection.DynamicLoadBalancingPolicy"/>
 
 	<!--  locking for a single node -->
-<!--	<bean name="lockManager"
+<!--	<bean orgAppName="lockManager"
         class="org.apache.usergrid.locking.singlenode.SingleNodeLockManagerImpl" />-->
 
 	<!--  hector based locks -->
@@ -86,10 +86,10 @@
 
 	<!--  zookeeper locks -->
 	<!--
-	<bean name="lockManager" class="org.apache.usergrid.locking.zookeeper.ZooKeeperLockManagerImpl"
>
-		<property name="hostPort" value="${zookeeper.url}"/>
-		<property name="sessionTimeout" value="2000"/>
-		<property name="maxAttempts" value="10"/>
+	<bean orgAppName="lockManager" class="org.apache.usergrid.locking.zookeeper.ZooKeeperLockManagerImpl"
>
+		<property orgAppName="hostPort" value="${zookeeper.url}"/>
+		<property orgAppName="sessionTimeout" value="2000"/>
+		<property orgAppName="maxAttempts" value="10"/>
 	</bean>  -->
 
 
@@ -196,7 +196,7 @@
     </bean>
 
     <bean id="jobFactory" class="org.apache.usergrid.batch.UsergridJobFactory" />
- 
+
 
     <context:component-scan base-package="org.apache.usergrid.batch.job" />
     <context:annotation-config />

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java
index 78a5137..10752d1 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java
@@ -25,6 +25,8 @@ import com.google.inject.assistedinject.Assisted;
 
 public interface EntityIndexFactory {
 
-    public EntityIndex createEntityIndex( 
+    public EntityIndex createEntityIndex(
         @Assisted ApplicationScope appScope);
+
+    void invalidate();
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/services/src/main/resources/usergrid-services-context.xml
----------------------------------------------------------------------
diff --git a/stack/services/src/main/resources/usergrid-services-context.xml b/stack/services/src/main/resources/usergrid-services-context.xml
index 666051d..8674b71 100644
--- a/stack/services/src/main/resources/usergrid-services-context.xml
+++ b/stack/services/src/main/resources/usergrid-services-context.xml
@@ -103,10 +103,10 @@
 
     <!--<bean id="importQueueListener" class="org.apache.usergrid.services.queues.ImportQueueListener"-->
           <!--scope="singleton">-->
-        <!--<constructor-arg name="emf" ref="entityManagerFactory" />-->
-        <!--<constructor-arg name="metricsService" ref="metricsFactory" />-->
-        <!--<constructor-arg name="smf" ref="serviceManagerFactory" />-->
-        <!--<constructor-arg name="props" ref="properties" />-->
+        <!--<constructor-arg orgAppName="emf" ref="entityManagerFactory" />-->
+        <!--<constructor-arg orgAppName="metricsService" ref="metricsFactory" />-->
+        <!--<constructor-arg orgAppName="smf" ref="serviceManagerFactory" />-->
+        <!--<constructor-arg orgAppName="props" ref="properties" />-->
 
     <!--</bean>-->
 


Mime
View raw message