Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 0D97B200B51 for ; Mon, 1 Aug 2016 19:56:10 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 0C1E3160A66; Mon, 1 Aug 2016 17:56:10 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id D0847160A6C for ; Mon, 1 Aug 2016 19:56:08 +0200 (CEST) Received: (qmail 14469 invoked by uid 500); 1 Aug 2016 17:56:08 -0000 Mailing-List: contact commits-help@atlas.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@atlas.incubator.apache.org Delivered-To: mailing list commits@atlas.incubator.apache.org Received: (qmail 14460 invoked by uid 99); 1 Aug 2016 17:56:08 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd1-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 01 Aug 2016 17:56:08 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd1-us-west.apache.org (ASF Mail Server at spamd1-us-west.apache.org) with ESMTP id 9F380C9FCA for ; Mon, 1 Aug 2016 17:56:07 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd1-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -4.465 X-Spam-Level: X-Spam-Status: No, score=-4.465 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-1.245] autolearn=disabled Received: from mx1-lw-eu.apache.org ([10.40.0.8]) by localhost (spamd1-us-west.apache.org [10.40.0.7]) (amavisd-new, port 10024) with ESMTP id 2cqznvhIxoql for ; Mon, 1 Aug 2016 17:56:05 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-eu.apache.org (ASF Mail Server at mx1-lw-eu.apache.org) with SMTP id 697D55F248 for ; Mon, 1 Aug 2016 17:56:04 +0000 (UTC) Received: (qmail 14378 invoked by uid 99); 1 Aug 2016 17:56:03 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 01 Aug 2016 17:56:03 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 9BA7FE058E; Mon, 1 Aug 2016 17:56:02 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: shwethags@apache.org To: commits@atlas.incubator.apache.org Date: Mon, 01 Aug 2016 17:56:02 -0000 Message-Id: <155155fbe65c424382a9f328ef8de324@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/2] incubator-atlas git commit: ATLAS-1026 StoreBackedTypeCache issues (dkantor via shwethags) archived-at: Mon, 01 Aug 2016 17:56:10 -0000 Repository: incubator-atlas Updated Branches: refs/heads/master 1c9b8b419 -> 69f6adfbc ATLAS-1026 StoreBackedTypeCache issues (dkantor via shwethags) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/96a11675 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/96a11675 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/96a11675 Branch: refs/heads/master Commit: 96a11675d83a0afc36e6893b6750abbb61a7c0fd Parents: 1c9b8b4 Author: Shwetha GS Authored: Mon Aug 1 22:55:43 2016 +0530 Committer: Shwetha GS Committed: Mon Aug 1 22:55:43 2016 +0530 ---------------------------------------------------------------------- release-log.txt | 1 + .../typestore/StoreBackedTypeCache.java | 23 ++---- .../typestore/StoreBackedTypeCacheTest.java | 32 +-------- ...StoreBackedTypeCacheMetadataServiceTest.java | 76 +++++++++++++++++--- .../atlas/typesystem/types/TypeSystem.java | 18 ++++- .../types/cache/DefaultTypeCache.java | 6 ++ .../atlas/typesystem/types/cache/TypeCache.java | 13 ++++ 7 files changed, 106 insertions(+), 63 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/96a11675/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index c21b43a..871a70c 100644 --- a/release-log.txt +++ b/release-log.txt @@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES: ALL CHANGES: +ATLAS-1026 StoreBackedTypeCache issues (dkantor via shwethags) ATLAS-861 1 table out of 50,000 tables is left unimported throwing exception during deserialization (sumasai via shwethags) ATLAS-1065 UI: Full text search view same as DSL's (kevalbhat18 via shwethags) ATLAS-1066 Falcon fails to post entity to Atlas due to kafka exception (mneethiraj via shwethags) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/96a11675/repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java b/repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java index d0c6f6a..600c3b4 100644 --- a/repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java +++ b/repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java @@ -106,12 +106,6 @@ public class StoreBackedTypeCache extends DefaultTypeCache { } } - @Override - public boolean has(String typeName) throws AtlasException { - - return (get(typeName) != null); - } - /** * Checks whether the specified type is cached in memory and does *not* * access the type store. Used for testing. @@ -124,21 +118,12 @@ public class StoreBackedTypeCache extends DefaultTypeCache { } /** - * Gets the requested type from the cache. - * This implementation will check the type store if the type is - * not already cached. If found in the type store, the type and - * any required super and attribute types are loaded from the type store, and - * added to the cache. - * - * @see org.apache.atlas.typesystem.types.cache.DefaultTypeCache#get(java.lang.String) + * Check the type store for the requested type. + * If found in the type store, the type and any required super and attribute types + * are loaded from the type store, and added to the cache. */ @Override - public IDataType get(String typeName) throws AtlasException { - - IDataType type = super.get(typeName); - if (type != null) { - return type; - } + public IDataType onTypeFault(String typeName) throws AtlasException { // Type is not cached - check the type store. // Any super and attribute types needed by the requested type http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/96a11675/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java b/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java index 2e3a3b1..b7cf7e9 100644 --- a/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java @@ -49,7 +49,6 @@ import java.util.Map; * Unit test for {@link StoreBackedTypeCache} */ @Guice(modules = RepositoryMetadataModule.class) -@Test(enabled = false) public class StoreBackedTypeCacheTest { @Inject @@ -106,6 +105,7 @@ public class StoreBackedTypeCacheTest { ts.reset(); } + @Test public void testGetClassType() throws Exception { for (Map.Entry typeEntry : classTypesToTest.entrySet()) { // Not cached yet @@ -122,20 +122,7 @@ public class StoreBackedTypeCacheTest { } } - public void testHasClassType() throws Exception { - for (Map.Entry typeEntry : classTypesToTest.entrySet()) { - // Not cached yet - Assert.assertFalse(typeCache.isCachedInMemory(typeEntry.getKey())); - - // Calling has() should result in type and its dependencies - // loaded from the type store and added to the cache. - Assert.assertTrue(typeCache.has(typeEntry.getKey())); - - // Verify the type is now cached in memory. - Assert.assertTrue(typeCache.isCachedInMemory(typeEntry.getKey())); - } - } - + @Test public void testGetTraitType() throws Exception { ImmutableList traitNames = ts.getTypeNamesByCategory(TypeCategory.TRAIT); for (String traitTypeName : traitNames) { @@ -153,21 +140,6 @@ public class StoreBackedTypeCacheTest { } } - public void testHasTraitType() throws Exception { - ImmutableList traitNames = ts.getTypeNamesByCategory(TypeCategory.TRAIT); - for (String traitTypeName : traitNames) { - // Not cached yet - Assert.assertFalse(typeCache.isCachedInMemory(traitTypeName)); - - // Calling has() should result in type and its dependencies - // loaded from the type store and added to the cache. - Assert.assertTrue(typeCache.has(traitTypeName)); - - // Verify the type is now cached. - Assert.assertTrue(typeCache.isCachedInMemory(traitTypeName)); - } - } - private void verifyHierarchicalType(T dataType, T expectedDataType) throws AtlasException { Assert.assertEquals(dataType.numFields, expectedDataType.numFields); Assert.assertEquals(dataType.immediateAttrs.size(), expectedDataType.immediateAttrs.size()); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/96a11675/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java b/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java index 5df1c8d..8fb59c5 100644 --- a/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java @@ -23,27 +23,37 @@ import org.apache.atlas.repository.typestore.ITypeStore; import org.apache.atlas.repository.typestore.StoreBackedTypeCache; import org.apache.atlas.repository.typestore.StoreBackedTypeCacheTestModule; import org.apache.atlas.services.MetadataService; +import org.apache.atlas.typesystem.TypesDef; +import org.apache.atlas.typesystem.json.TypesSerialization; +import org.apache.atlas.typesystem.types.AttributeDefinition; +import org.apache.atlas.typesystem.types.ClassType; +import org.apache.atlas.typesystem.types.DataTypes; +import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; +import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.TypeSystem; +import org.apache.atlas.typesystem.types.TypeUpdateException; import org.apache.atlas.typesystem.types.cache.TypeCache; +import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Guice; import org.testng.annotations.Test; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.util.TitanCleanup; /** - * Verify MetadataService type lookup triggers StoreBackedTypeCache to load type from the store. - * StoreBackedTypeCacheTestModule Guice module uses Atlas configuration - * which has type cache implementation class set to {@link StoreBackedTypeCache}. + * Verify MetadataService type operations trigger StoreBackedTypeCache to load non-cached types from the store. + * StoreBackedTypeCacheTestModule Guice module sets Atlas configuration + * to use {@link StoreBackedTypeCache} as the TypeCache implementation class. */ @Guice(modules = StoreBackedTypeCacheTestModule.class) -@Test(enabled = false) public class StoreBackedTypeCacheMetadataServiceTest { @Inject @@ -55,13 +65,18 @@ public class StoreBackedTypeCacheMetadataServiceTest @Inject TypeCache typeCache; + private StoreBackedTypeCache storeBackedTypeCache; + @Inject private GraphProvider graphProvider; private TypeSystem ts; @BeforeClass - public void setUp() throws Exception { + public void oneTimeSetup() throws Exception { + Assert.assertTrue(typeCache instanceof StoreBackedTypeCache); + storeBackedTypeCache = (StoreBackedTypeCache) typeCache; + ts = TypeSystem.getInstance(); ts.reset(); @@ -70,6 +85,10 @@ public class StoreBackedTypeCacheMetadataServiceTest TestUtils.createHiveTypes(ts); ImmutableList typeNames = ts.getTypeNames(); typeStore.store(ts, typeNames); + } + + @BeforeMethod + public void setUp() throws Exception { ts.reset(); } @@ -91,16 +110,51 @@ public class StoreBackedTypeCacheMetadataServiceTest } } - public void testIt() throws Exception { - Assert.assertTrue(typeCache instanceof StoreBackedTypeCache); - StoreBackedTypeCache storeBackedCache = (StoreBackedTypeCache) typeCache; - + @Test + public void testGetTypeDefinition() throws Exception { // Cache should be empty - Assert.assertFalse(storeBackedCache.isCachedInMemory("Manager")); + Assert.assertFalse(storeBackedTypeCache.isCachedInMemory("Manager")); // Type lookup on MetadataService should cause Manager type to be loaded from the type store // and cached. Assert.assertNotNull(metadataService.getTypeDefinition("Manager")); - Assert.assertTrue(storeBackedCache.isCachedInMemory("Manager")); + Assert.assertTrue(storeBackedTypeCache.isCachedInMemory("Manager")); + } + + @Test + public void testValidUpdateType() throws Exception { + // Cache should be empty + Assert.assertFalse(storeBackedTypeCache.isCachedInMemory(TestUtils.TABLE_TYPE)); + + TypesDef typesDef = TestUtils.defineHiveTypes(); + String json = TypesSerialization.toJson(typesDef); + + // Update types with same definition, which should succeed. + metadataService.updateType(json); + + // hive_table type should now be cached. + Assert.assertTrue(storeBackedTypeCache.isCachedInMemory(TestUtils.TABLE_TYPE)); + } + + @Test + public void testInvalidUpdateType() throws Exception { + // Cache should be empty + Assert.assertFalse(storeBackedTypeCache.isCachedInMemory(TestUtils.TABLE_TYPE)); + + HierarchicalTypeDefinition classTypeDef = TypesUtil.createClassTypeDef(TestUtils.TABLE_TYPE, ImmutableSet.of(), + new AttributeDefinition("attr1", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null)); + String json = TypesSerialization.toJson(classTypeDef, false); + + // Try to update the type with disallowed changes. Should fail with TypeUpdateException. + try { + metadataService.updateType(json); + Assert.fail(TypeUpdateException.class.getSimpleName() + " was expected but none thrown"); + } + catch(TypeUpdateException e) { + // good + } + + // hive_table type should now be cached. + Assert.assertTrue(storeBackedTypeCache.isCachedInMemory(TestUtils.TABLE_TYPE)); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/96a11675/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java index 70f9ea5..70ba89b 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java @@ -148,7 +148,6 @@ public class TypeSystem { } public T getDataType(Class cls, String name) throws AtlasException { - if (isCoreType(name)) { return cls.cast(coreTypes.get(name)); } @@ -180,6 +179,14 @@ public class TypeSystem { return cls.cast(dT); } + /* + * Invoke cache callback to possibly obtain type from other storage. + */ + IDataType dT = typeCache.onTypeFault(name); + if (dT != null) { + return cls.cast(dT); + } + throw new TypeNotFoundException(String.format("Unknown datatype: %s", name)); } @@ -599,8 +606,13 @@ public class TypeSystem { private void validateUpdateIsPossible() throws TypeUpdateException, AtlasException { //If the type is modified, validate that update can be done for (IDataType newType : transientTypes.values()) { - if (TypeSystem.this.isRegistered(newType.getName())) { - IDataType oldType = TypeSystem.this.typeCache.get(newType.getName()); + IDataType oldType = null; + try { + oldType = TypeSystem.this.getDataType(IDataType.class, newType.getName()); + } catch (TypeNotFoundException e) { + LOG.debug("No existing type %s found - update OK", newType.getName()); + } + if (oldType != null) { oldType.validateUpdate(newType); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/96a11675/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java index 6c6d5a0..ce750af 100644 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java @@ -288,4 +288,10 @@ public class DefaultTypeCache implements TypeCache { types_.clear(); } + + @Override + public IDataType onTypeFault(String typeName) throws AtlasException { + + return null; + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/96a11675/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java index 2bcbdd7..87d83a6 100644 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java @@ -21,6 +21,7 @@ package org.apache.atlas.typesystem.types.cache; import org.apache.atlas.AtlasException; import org.apache.atlas.typesystem.types.DataTypes; import org.apache.atlas.typesystem.types.IDataType; +import org.apache.atlas.typesystem.types.TypeSystem; import java.util.Collection; import java.util.Map; @@ -137,4 +138,16 @@ public interface TypeCache { * */ void clear(); + + /** + * Called when a type lookup request on {@link TypeSystem} + * fails because the type is not present in the runtime type information. + * Implementations can take action such as retrieving the requested type + * from some persistent storage. + + * @param typeName + * @throws AtlasException + */ + IDataType onTypeFault(String typeName) throws AtlasException; + }