Return-Path: X-Original-To: apmail-usergrid-commits-archive@minotaur.apache.org Delivered-To: apmail-usergrid-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id B0FC3178AB for ; Thu, 12 Mar 2015 22:41:09 +0000 (UTC) Received: (qmail 80257 invoked by uid 500); 12 Mar 2015 22:41:09 -0000 Delivered-To: apmail-usergrid-commits-archive@usergrid.apache.org Received: (qmail 80128 invoked by uid 500); 12 Mar 2015 22:41:09 -0000 Mailing-List: contact commits-help@usergrid.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@usergrid.incubator.apache.org Delivered-To: mailing list commits@usergrid.incubator.apache.org Received: (qmail 79774 invoked by uid 99); 12 Mar 2015 22:41:09 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 12 Mar 2015 22:41:09 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 29A32E190B; Thu, 12 Mar 2015 22:41:09 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: grey@apache.org To: commits@usergrid.apache.org Date: Thu, 12 Mar 2015 22:41:22 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [15/38] incubator-usergrid git commit: Added caching to org/app lookup Added caching to org/app lookup Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4c8727c1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4c8727c1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4c8727c1 Branch: refs/heads/USERGRID-396 Commit: 4c8727c1dc0e5f02a498849be1a19e6342f6accf Parents: 2c54a2b Author: Todd Nine Authored: Fri Mar 6 15:13:49 2015 -0700 Committer: Todd Nine Committed: Fri Mar 6 15:13:49 2015 -0700 ---------------------------------------------------------------------- .../corepersistence/CpEntityManager.java | 1 + .../corepersistence/CpEntityManagerFactory.java | 116 ++++-------- .../corepersistence/OrgApplicationCache.java | 67 +++++++ .../OrgApplicationCacheImpl.java | 181 +++++++++++++++++++ .../main/resources/usergrid-core-context.xml | 14 +- .../persistence/index/EntityIndexFactory.java | 4 +- .../resources/usergrid-services-context.xml | 8 +- 7 files changed, 296 insertions(+), 95 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java index 5619382..652e084 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java @@ -676,6 +676,7 @@ public class CpEntityManager implements EntityManager { @Override public RelationManager getRelationManager( EntityRef entityRef ) { + Preconditions.checkNotNull( entityRef, "entityRef cannot be null" ); CpRelationManager rmi = new CpRelationManager(); rmi.init( this, emf, applicationId, entityRef, null ); return rmi; http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java index 9aa36a0..16de53e 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java @@ -15,6 +15,7 @@ */ package org.apache.usergrid.corepersistence; +import com.google.common.base.Optional; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; @@ -98,6 +99,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application } }); + private final OrgApplicationCache orgApplicationCache; + private ManagerCache managerCache; private DataMigrationManager dataMigrationManager; @@ -116,7 +119,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application this.managerCache = injector.getInstance( ManagerCache.class ); this.dataMigrationManager = injector.getInstance( DataMigrationManager.class ); - + this.orgApplicationCache = new OrgApplicationCacheImpl( this ); } @@ -156,18 +159,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application } -// public ManagerCache getManagerCache() { -// -// if ( managerCache == null ) { -// managerCache = injector.getInstance( ManagerCache.class ); -// -// dataMigrationManager = injector.getInstance( DataMigrationManager.class ); -// } -// return managerCache; -// } - - - @Override public EntityManager getEntityManager(UUID applicationId) { try { @@ -207,15 +198,16 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application String appName = buildAppName( orgName, name ); - UUID applicationId = lookupApplication( appName ); - if ( applicationId != null ) { + final Optional appId = orgApplicationCache.getApplicationId( appName ); + + if ( appId.isPresent() ) { throw new ApplicationAlreadyExistsException( name ); } - applicationId = UUIDGenerator.newTimeUUID(); + UUID applicationId = UUIDGenerator.newTimeUUID(); - logger.debug( "New application orgName {} name {} id {} ", + logger.debug( "New application orgName {} orgAppName {} id {} ", new Object[] { orgName, name, applicationId.toString() } ); initializeApplication( orgName, applicationId, appName, properties ); @@ -233,6 +225,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application Map properties ) throws Exception { + + //Ensure our management system exists before creating our application + init(); + EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID); final String appName = buildAppName( organizationName, name ); @@ -244,8 +240,14 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application getSetup().setupApplicationKeyspace( applicationId, appName ); - UUID orgUuid = lookupOrganization( organizationName ); - if ( orgUuid == null ) { + + final Optional cachedValue = orgApplicationCache.getOrganizationId( name ); + + + final UUID orgUuid; + + if ( !cachedValue.isPresent() ) { + // create new org because the specified one does not exist final String orgName = organizationName; @@ -263,6 +265,11 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application em.refreshIndex(); orgUuid = orgInfo.getUuid(); + + //evit so it's re-loaded later + orgApplicationCache.evictOrgId( name ); + } else{ + orgUuid = cachedValue.get(); } // create appinfo entry in the system app @@ -293,6 +300,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application appEm.refreshIndex(); logger.info("Initialized application {}", appName ); + + //evict app Id from cache + orgApplicationCache.evictAppId( appName ); + return applicationId; } @@ -374,71 +385,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application } - public UUID lookupOrganization( String name ) throws Exception { - init(); - - - // Query q = Query.fromQL(PROPERTY_NAME + " = '" + name + "'"); - EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID ); - - - final EntityRef alias = em.getAlias( "organizations", name ); - - if ( alias == null ) { - return null; - } - - final Entity entity = em.get( alias ); - - if ( entity == null ) { - return null; - } - - return entity.getUuid(); - // Results results = em.searchCollection( em.getApplicationRef(), "organizations", q ); - // - // if ( results.isEmpty() ) { - // return null; - // } - // - // return results.iterator().next().getUuid(); - } - @Override - public UUID lookupApplication( String name ) throws Exception { - init(); - - // TODO: why does this not work for restored apps - -// EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID ); -// final EntityRef alias = em.getAlias( CpNamingUtils.APPINFOS, name ); -// if ( alias == null ) { -// return null; -// } -// final Entity entity = em.get( alias ); -// if ( entity == null ) { -// return null; -// } -// final UUID property = ( UUID ) entity.getProperty( "applicationUuid" ); -// return property; - - Query q = Query.fromQL( PROPERTY_NAME + " = '" + name + "'"); - - EntityManager em = getEntityManager(CpNamingUtils.SYSTEM_APP_ID); - - Results results = em.searchCollection( em.getApplicationRef(), "appinfos", q); - - if ( results.isEmpty() ) { - return null; - } - - Entity entity = results.iterator().next(); - Object uuidObject = entity.getProperty("applicationUuid"); - if ( uuidObject instanceof UUID ) { - return (UUID)uuidObject; - } - return UUIDUtils.tryExtractUUID( entity.getProperty("applicationUuid").toString() ); + public UUID lookupApplication( String orgAppName ) throws Exception { + return orgApplicationCache.getApplicationId( orgAppName ).orNull(); } @@ -632,7 +582,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application em.update( propsEntity ); } catch (Exception ex) { - logger.error("Error deleting service property name: " + name, ex); + logger.error("Error deleting service property orgAppName: " + name, ex); return false; } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCache.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCache.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCache.java new file mode 100644 index 0000000..b20dbe1 --- /dev/null +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCache.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.usergrid.corepersistence; + + +import java.util.UUID; + +import org.apache.usergrid.persistence.Entity; + +import com.google.common.base.Optional; + + +/** + * A simple cache interface for looking up entities from an EM + */ +public interface OrgApplicationCache { + + + /** + * Get an entity by it's alias property. The result is cached. To clear it call evict or evict all + * @param + * @return + */ + public Optional getOrganizationId(final String orgName); + + /** + * Evict the org by name + * @param orgName + */ + public void evictOrgId(final String orgName); + + /** + * Evict the application by name + * @param applicationName + * @return + */ + public Optional getApplicationId(final String applicationName); + + + /** + * Evict the app id by the name + */ + public void evictAppId(final String applicationname); + + + /** + * Evict all caches + */ + public void evictAll(); +} http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java new file mode 100644 index 0000000..4baf598 --- /dev/null +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.usergrid.corepersistence; + + +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import org.apache.usergrid.corepersistence.util.CpNamingUtils; +import org.apache.usergrid.persistence.Entity; +import org.apache.usergrid.persistence.EntityManager; +import org.apache.usergrid.persistence.EntityManagerFactory; +import org.apache.usergrid.persistence.EntityRef; +import org.apache.usergrid.persistence.Results; +import org.apache.usergrid.persistence.index.query.Query; +import org.apache.usergrid.utils.UUIDUtils; + +import com.google.common.base.Optional; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; + +import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME; + + +/** + * Implements the org app cache for faster runtime lookups. These values are immutable, so this LRU cache can stay + * full for the duration of the execution + */ +public class OrgApplicationCacheImpl implements OrgApplicationCache { + + + /** + * Cache the pointer to our root entity manager for reference + */ + private final EntityManager rootEm; + + private final LoadingCache> orgCache = + CacheBuilder.newBuilder().maximumSize( 10000 ).build( new CacheLoader>() { + @Override + public Optional load( final String key ) throws Exception { + return fetchOrganizationId( key ); + } + } ); + + + private final LoadingCache> appCache = + CacheBuilder.newBuilder().maximumSize( 10000 ).build( new CacheLoader>() { + @Override + public Optional load( final String key ) throws Exception { + return fetchApplicationId( key ); + } + } ); + + + public OrgApplicationCacheImpl( final EntityManagerFactory emf ) { + this.rootEm = emf.getEntityManager( CpNamingUtils.SYSTEM_APP_ID ); + } + + + @Override + public Optional getOrganizationId( final String orgName ) { + try { + return orgCache.get( orgName ); + } + catch ( ExecutionException e ) { + throw new RuntimeException( "Unable to load org cache", e ); + } + } + + + /** + * Fetches the organization + */ + private Optional fetchOrganizationId( final String orgName ) { + + try { + final EntityRef alias = rootEm.getAlias( "organizations", orgName ); + + if ( alias == null ) { + return Optional.absent(); + } + + final Entity entity; + + entity = rootEm.get( alias ); + + + if ( entity == null ) { + return Optional.absent(); + } + + return Optional.of( entity.getUuid() ); + } + catch ( Exception e ) { + throw new RuntimeException( "Unable to load organization Id for caching", e ); + } + } + + + @Override + public void evictOrgId( final String orgName ) { + orgCache.invalidate( orgName ); + } + + + @Override + public Optional getApplicationId( final String applicationName ) { + try { + return appCache.get( applicationName ); + } + catch ( ExecutionException e ) { + throw new RuntimeException( "Unable to load org cache", e ); + } + } + + + /** + * Fetch our application id + */ + private Optional fetchApplicationId( final String applicationName ) { + + try { + Query q = Query.fromQL( PROPERTY_NAME + " = '" + applicationName + "'" ); + + + Results results = rootEm.searchCollection( rootEm.getApplicationRef(), "appinfos", q ); + + if ( results.isEmpty() ) { + return Optional.absent(); + } + + Entity entity = results.iterator().next(); + Object uuidObject = entity.getProperty( "applicationUuid" ); + + final UUID value; + if ( uuidObject instanceof UUID ) { + value = ( UUID ) uuidObject; + } + else { + value = UUIDUtils.tryExtractUUID( entity.getProperty( "applicationUuid" ).toString() ); + } + + + return Optional.of( value ); + } + catch ( Exception e ) { + throw new RuntimeException( "Unable to retreive application id", e ); + } + } + + + @Override + public void evictAppId( final String applicationName ) { + appCache.invalidate( applicationName ); + } + + + @Override + public void evictAll() { + orgCache.invalidateAll(); + appCache.invalidateAll(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/resources/usergrid-core-context.xml ---------------------------------------------------------------------- diff --git a/stack/core/src/main/resources/usergrid-core-context.xml b/stack/core/src/main/resources/usergrid-core-context.xml index c8ce0d8..cd40d6d 100644 --- a/stack/core/src/main/resources/usergrid-core-context.xml +++ b/stack/core/src/main/resources/usergrid-core-context.xml @@ -54,7 +54,7 @@ - + @@ -70,7 +70,7 @@ - @@ -86,10 +86,10 @@ @@ -196,7 +196,7 @@ - + http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java index 78a5137..10752d1 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java @@ -25,6 +25,8 @@ import com.google.inject.assistedinject.Assisted; public interface EntityIndexFactory { - public EntityIndex createEntityIndex( + public EntityIndex createEntityIndex( @Assisted ApplicationScope appScope); + + void invalidate(); } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/services/src/main/resources/usergrid-services-context.xml ---------------------------------------------------------------------- diff --git a/stack/services/src/main/resources/usergrid-services-context.xml b/stack/services/src/main/resources/usergrid-services-context.xml index 666051d..8674b71 100644 --- a/stack/services/src/main/resources/usergrid-services-context.xml +++ b/stack/services/src/main/resources/usergrid-services-context.xml @@ -103,10 +103,10 @@ - - - - + + + +