atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shweth...@apache.org
Subject incubator-atlas git commit: ATLAS-1184 ReservedTypesRegistrar checks for existence of 1st class type (svimal2106 via shwethags)
Date Thu, 29 Sep 2016 16:31:20 GMT
Repository: incubator-atlas
Updated Branches:
  refs/heads/master 100749b49 -> 4c56c61f8


ATLAS-1184 ReservedTypesRegistrar checks for existence of 1st class type (svimal2106 via shwethags)


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/4c56c61f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/4c56c61f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/4c56c61f

Branch: refs/heads/master
Commit: 4c56c61f85c4c38a1d2bec3088130f2ea5f1e8fe
Parents: 100749b
Author: Shwetha GS <sshivalingamurthy@hortonworks.com>
Authored: Thu Sep 29 11:50:47 2016 +0530
Committer: Shwetha GS <sshivalingamurthy@hortonworks.com>
Committed: Thu Sep 29 11:50:47 2016 +0530

----------------------------------------------------------------------
 release-log.txt                                 |  1 +
 .../atlas/services/ReservedTypesRegistrar.java  | 55 ++++++++++++++++----
 .../test/java/org/apache/atlas/TestUtils.java   | 49 +++++++++++++++++
 .../services/ReservedTypesRegistrarTest.java    | 32 ++++++------
 4 files changed, 112 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4c56c61f/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 0d78273..1a691ca 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements
for al
 ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
 
 ALL CHANGES:
+ATLAS-1184 ReservedTypesRegistrar checks for existence of 1st class type (svimal2106 via
shwethags)
 ATLAS-1199 Atlas UI not loading after fresh build due to jquery-asBreadcrumbs plugin upgrade
(kevalbhatt via shwethags)
 ATLAS-1174 Framework to apply updates to types in the type-system (sarath.kum4r@gmail.com
via shwethags)
 ATLAS-1155 Errors in Eclipse when I bring in the latest code (davidrad via shwethags)

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4c56c61f/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java
b/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java
index 41c0155..6e0b586 100644
--- a/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java
+++ b/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java
@@ -18,12 +18,12 @@
 
 package org.apache.atlas.services;
 
+import com.google.common.collect.ImmutableList;
 import org.apache.atlas.AtlasException;
 import org.apache.atlas.typesystem.TypesDef;
 import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.typesystem.types.ClassType;
-import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.*;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -31,8 +31,10 @@ import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
+import java.util.List;
 
 public class ReservedTypesRegistrar implements IBootstrapTypesRegistrar {
 
@@ -81,13 +83,46 @@ public class ReservedTypesRegistrar implements IBootstrapTypesRegistrar
{
             LOG.error("Error while deserializing JSON in {}", typeDefName);
             throw new ReservedTypesRegistrationException("Error while deserializing JSON
in " + typeDefName, e);
         }
-        HierarchicalTypeDefinition<ClassType> classDef = typesDef.classTypesAsJavaList().get(0);
-        if (!typeSystem.isRegistered(classDef.typeName)) {
-            metadataService.createType(typeDefJSON);
-            LOG.info("Registered types in {}", typeDefName);
-        } else {
-            LOG.warn("class {} already registered, ignoring types in {}", classDef.typeName,
-                    typeDefName);
+        List<HierarchicalTypeDefinition<ClassType>> createClassDefList = new
ArrayList<>();
+        List<HierarchicalTypeDefinition<TraitType>> createTraitDefList = new
ArrayList<>();
+        List<EnumTypeDefinition> createEnumDefList = new ArrayList<>();
+        List<StructTypeDefinition> createStructDefList = new ArrayList<>();
+
+        for(HierarchicalTypeDefinition<ClassType> classTypeDef:typesDef.classTypesAsJavaList()){
+            if(!typeSystem.isRegistered(classTypeDef.typeName)){
+                LOG.debug("ClassType {} is not registered. Adding to create type list", classTypeDef.typeName);
+                createClassDefList.add(classTypeDef);
+            }
+        }
+
+        for(HierarchicalTypeDefinition<TraitType> traitTypeDef:typesDef.traitTypesAsJavaList()){
+            if(!typeSystem.isRegistered(traitTypeDef.typeName)){
+                LOG.debug("TraitType {} is not registered. Adding to create type list", traitTypeDef.typeName);
+                createTraitDefList.add(traitTypeDef);
+            }
+        }
+
+        for(StructTypeDefinition structTypeDef:typesDef.structTypesAsJavaList()){
+            if(!typeSystem.isRegistered(structTypeDef.typeName)){
+                LOG.debug("StructType {} is not registered. Adding to create type list",
structTypeDef.typeName);
+                createStructDefList.add(structTypeDef);
+            }
+        }
+
+        for(EnumTypeDefinition enumTypeDef:typesDef.enumTypesAsJavaList()){
+            if(!typeSystem.isRegistered(enumTypeDef.name)){
+                LOG.debug("EnumType {} is not registered. Adding to create type list", enumTypeDef.name);
+                createEnumDefList.add(enumTypeDef);
+            }
+        }
+
+        TypesDef createTypes = TypesUtil.getTypesDef(ImmutableList.copyOf(createEnumDefList),
ImmutableList.copyOf(createStructDefList),
+                ImmutableList.copyOf(createTraitDefList), ImmutableList.copyOf(createClassDefList));
+
+        String createTypeJSON = TypesSerialization.toJson(createTypes);
+        if(createTypeJSON != null) {
+            metadataService.createType(createTypeJSON);
+            LOG.info("Created types definition JSON {}", createTypeJSON);
         }
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4c56c61f/repository/src/test/java/org/apache/atlas/TestUtils.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/TestUtils.java b/repository/src/test/java/org/apache/atlas/TestUtils.java
index bd9df62..b27854e 100755
--- a/repository/src/test/java/org/apache/atlas/TestUtils.java
+++ b/repository/src/test/java/org/apache/atlas/TestUtils.java
@@ -239,6 +239,55 @@ public final class TestUtils {
 
     public static final String NAME = "name";
 
+    public static TypesDef simpleType(){
+        HierarchicalTypeDefinition<ClassType> superTypeDefinition =
+                createClassTypeDef("h_type", ImmutableSet.<String>of(),
+                        createOptionalAttrDef("attr", DataTypes.STRING_TYPE));
+
+        StructTypeDefinition structTypeDefinition = new StructTypeDefinition("s_type", "structType",
+                new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE)});
+
+        HierarchicalTypeDefinition<TraitType> traitTypeDefinition =
+                createTraitTypeDef("t_type", "traitType", ImmutableSet.<String>of());
+
+        EnumValue values[] = {new EnumValue("ONE", 1),};
+
+        EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("e_type", "enumType",
values);
+        return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.of(structTypeDefinition),
+                ImmutableList.of(traitTypeDefinition), ImmutableList.of(superTypeDefinition));
+    }
+
+    public static TypesDef simpleTypeUpdated(){
+        HierarchicalTypeDefinition<ClassType> superTypeDefinition =
+                createClassTypeDef("h_type", ImmutableSet.<String>of(),
+                        createOptionalAttrDef("attr", DataTypes.STRING_TYPE));
+
+        HierarchicalTypeDefinition<ClassType> newSuperTypeDefinition =
+                createClassTypeDef("new_h_type", ImmutableSet.<String>of(),
+                        createOptionalAttrDef("attr", DataTypes.STRING_TYPE));
+
+        StructTypeDefinition structTypeDefinition = new StructTypeDefinition("s_type", "structType",
+                new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE)});
+
+        HierarchicalTypeDefinition<TraitType> traitTypeDefinition =
+                createTraitTypeDef("t_type", "traitType", ImmutableSet.<String>of());
+
+        EnumValue values[] = {new EnumValue("ONE", 1),};
+
+        EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("e_type", "enumType",
values);
+        return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.of(structTypeDefinition),
+                ImmutableList.of(traitTypeDefinition), ImmutableList.of(superTypeDefinition,
newSuperTypeDefinition));
+    }
+
+    public static TypesDef simpleTypeUpdatedDiff() {
+        HierarchicalTypeDefinition<ClassType> newSuperTypeDefinition =
+                createClassTypeDef("new_h_type", ImmutableSet.<String>of(),
+                        createOptionalAttrDef("attr", DataTypes.STRING_TYPE));
+
+        return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
+                ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), ImmutableList.of(newSuperTypeDefinition));
+    }
+
     public static TypesDef defineHiveTypes() {
         String _description = "_description";
         HierarchicalTypeDefinition<ClassType> superTypeDefinition =

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4c56c61f/repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java
b/repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java
index d602bd8..effab15 100644
--- a/repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java
+++ b/repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java
@@ -23,6 +23,8 @@ import org.apache.atlas.TestUtils;
 import org.apache.atlas.typesystem.TypesDef;
 import org.apache.atlas.typesystem.json.TypesSerialization;
 import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.TypeUtils;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
 import org.mockito.InOrder;
 import org.mockito.Mock;
 import org.mockito.MockitoAnnotations;
@@ -55,17 +57,6 @@ public class ReservedTypesRegistrarTest {
     }
 
     @Test
-    public void testRegisterFirstChecksClassTypeIsRegistered() throws AtlasException {
-        ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar();
-        TypesDef typesDef = TestUtils.defineHiveTypes();
-        String typesJson = TypesSerialization.toJson(typesDef);
-        reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json",
typesJson);
-        InOrder inOrder = inOrder(typeSystem, metadataService);
-        inOrder.verify(typeSystem).isRegistered(typesDef.classTypesAsJavaList().get(0).typeName);
-        inOrder.verify(metadataService).createType(typesJson);
-    }
-
-    @Test
     public void testRegisterCreatesTypesUsingMetadataService() throws AtlasException {
         ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar();
         TypesDef typesDef = TestUtils.defineHiveTypes();
@@ -90,12 +81,23 @@ public class ReservedTypesRegistrarTest {
     }
 
     @Test
-    public void testShouldNotRegisterIfTypeIsAlreadyRegistered() throws AtlasException {
+    public void testCreateAndUpdateType() throws AtlasException{
         ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar();
-        TypesDef typesDef = TestUtils.defineHiveTypes();
+        TypesDef typesDef = TestUtils.simpleType();
         String typesJson = TypesSerialization.toJson(typesDef);
-        when(typeSystem.isRegistered(typesDef.classTypesAsJavaList().get(0).typeName)).thenReturn(true);
         reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json",
typesJson);
-        verifyZeroInteractions(metadataService);
+        verify(metadataService).createType(typesJson);
+
+        //test update simple type
+        TypesDef updatedTypesDef = TestUtils.simpleTypeUpdated();
+        String updatedTypesJson = TypesSerialization.toJson(updatedTypesDef);
+        TypesDef simpleTypeUpdatedDiff = TestUtils.simpleTypeUpdatedDiff();
+        String simpleTypeUpdatedDiffJson = TypesSerialization.toJson(simpleTypeUpdatedDiff);
+        when(typeSystem.isRegistered("h_type")).thenReturn(true);
+        when(typeSystem.isRegistered("t_type")).thenReturn(true);
+        when(typeSystem.isRegistered("s_type")).thenReturn(true);
+        when(typeSystem.isRegistered("e_type")).thenReturn(true);
+        reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json",
updatedTypesJson);
+        verify(metadataService).createType(simpleTypeUpdatedDiffJson);
     }
 }


Mime
View raw message