atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From suma...@apache.org
Subject [1/2] incubator-atlas git commit: ATLAS-171 Ability to update type definition(shwethags via sumasai)
Date Thu, 03 Dec 2015 05:50:51 GMT
Repository: incubator-atlas
Updated Branches:
  refs/heads/master 919120f65 -> bf5672c54


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeUpdateBaseTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeUpdateBaseTest.java
b/typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeUpdateBaseTest.java
new file mode 100644
index 0000000..4a6ed2d
--- /dev/null
+++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeUpdateBaseTest.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.testng.Assert;
+
+public abstract class TypeUpdateBaseTest extends BaseTest {
+    protected void testTypeUpdateForAttributes() throws Exception {
+        StructTypeDefinition typeDefinition =
+                getTypeDefinition(newName(), TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE));
+        TypeSystem ts = getTypeSystem();
+        TypesDef typesDef = getTypesDef(typeDefinition);
+        ts.defineTypes(typesDef);
+        String typeName = typeDefinition.typeName;
+
+        //Allow modifying required to optional attribute
+        typeDefinition = getTypeDefinition(typeName, TypesUtil.createOptionalAttrDef("a",
DataTypes.INT_TYPE));
+        ts.updateTypes(getTypesDef(typeDefinition));
+
+        //Allow adding new optional attribute
+        typeDefinition = getTypeDefinition(typeName, TypesUtil.createOptionalAttrDef("a",
DataTypes.INT_TYPE),
+                TypesUtil.createOptionalAttrDef("b", DataTypes.INT_TYPE));
+        ts.updateTypes(getTypesDef(typeDefinition));
+
+        //Don't allow adding required attribute
+        typeDefinition = getTypeDefinition(typeName, TypesUtil.createOptionalAttrDef("a",
DataTypes.INT_TYPE),
+                TypesUtil.createOptionalAttrDef("b", DataTypes.INT_TYPE),
+                TypesUtil.createRequiredAttrDef("c", DataTypes.INT_TYPE));
+        try {
+            ts.updateTypes(getTypesDef(typeDefinition));
+            Assert.fail("Expected TypeUpdateException");
+        } catch (TypeUpdateException e) {
+            //assert that type is not updated when validation fails
+            Assert.assertEquals(getNumberOfFields(ts, typeDefinition.typeName), 2);
+        }
+
+        //Don't allow removing attribute
+        typeDefinition = getTypeDefinition(typeName, TypesUtil.createOptionalAttrDef("a",
DataTypes.INT_TYPE));
+        try {
+            ts.updateTypes(getTypesDef(typeDefinition));
+        } catch (TypeUpdateException e) {
+            //expected
+        }
+
+        //Don't allow modifying other fields of attribute definition - optional to required
+        typeDefinition = getTypeDefinition(typeName, TypesUtil.createOptionalAttrDef("a",
DataTypes.INT_TYPE),
+                TypesUtil.createRequiredAttrDef("b", DataTypes.INT_TYPE));
+        try {
+            ts.updateTypes(getTypesDef(typeDefinition));
+        } catch (TypeUpdateException e) {
+            //expected
+        }
+
+        //Don't allow modifying other fields of attribute definition - attribute type change
+        typeDefinition = getTypeDefinition(typeName, TypesUtil.createOptionalAttrDef("a",
DataTypes.INT_TYPE),
+                TypesUtil.createOptionalAttrDef("b", DataTypes.STRING_TYPE));
+        try {
+            ts.updateTypes(getTypesDef(typeDefinition));
+        } catch (TypeUpdateException e) {
+            //expected
+        }
+
+        //Don't allow modifying other fields of attribute definition - attribute type change
+        typeDefinition = getTypeDefinition(typeName, TypesUtil.createRequiredAttrDef("a",
DataTypes.INT_TYPE),
+                new AttributeDefinition("b", DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()),
+                        Multiplicity.COLLECTION, false, null));
+        try {
+            ts.updateTypes(getTypesDef(typeDefinition));
+        } catch (TypeUpdateException e) {
+            //expected
+        }
+    }
+
+    protected abstract int getNumberOfFields(TypeSystem ts, String typeName) throws Exception;
+
+    protected abstract TypesDef getTypesDef(StructTypeDefinition typeDefinition);
+
+    protected abstract StructTypeDefinition getTypeDefinition(String typeName,
+                                                              AttributeDefinition... attributeDefinitions);
+}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java b/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java
index 7b8f88c..1251062 100755
--- a/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java
+++ b/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java
@@ -35,7 +35,6 @@ import org.apache.atlas.typesystem.types.IDataType;
 import org.apache.atlas.typesystem.types.Multiplicity;
 import org.apache.atlas.typesystem.types.StructTypeDefinition;
 import org.apache.atlas.typesystem.types.TraitType;
-import org.apache.atlas.typesystem.types.TypeUtils;
 import org.apache.atlas.typesystem.types.utils.TypesUtil;
 import org.codehaus.jettison.json.JSONArray;
 
@@ -153,7 +152,7 @@ public class QuickStart {
 
         HierarchicalTypeDefinition<TraitType> jdbcTraitDef = TypesUtil.createTraitTypeDef("JdbcAccess",
null);
 
-        return TypeUtils.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
+        return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
                 ImmutableList.of(dimTraitDef, factTraitDef, piiTraitDef, metricTraitDef,
etlTraitDef, jdbcTraitDef),
                 ImmutableList.of(dbClsDef, storageDescClsDef, columnClsDef, tblClsDef, loadProcessClsDef,
viewClsDef));
     }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java
index c049350..1f0b98a 100755
--- a/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java
@@ -38,6 +38,7 @@ import javax.ws.rs.Consumes;
 import javax.ws.rs.DefaultValue;
 import javax.ws.rs.GET;
 import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
@@ -110,6 +111,50 @@ public class TypesResource {
     }
 
     /**
+     * Update of existing types - if the given type doesn't exist, creates new type
+     * Allowed updates are:
+     * 1. Add optional attribute
+     * 2. Change required to optional attribute
+     * 3. Add super types - super types shouldn't contain any required attributes
+     * @param request
+     * @return
+     */
+    @PUT
+    @Consumes(Servlets.JSON_MEDIA_TYPE)
+    @Produces(Servlets.JSON_MEDIA_TYPE)
+    public Response update(@Context HttpServletRequest request) {
+        try {
+            final String typeDefinition = Servlets.getRequestPayload(request);
+            LOG.debug("Updating type with definition {} ", typeDefinition);
+
+            JSONObject typesJson = metadataService.updateType(typeDefinition);
+            final JSONArray typesJsonArray = typesJson.getJSONArray(AtlasClient.TYPES);
+
+            JSONArray typesResponse = new JSONArray();
+            for (int i = 0; i < typesJsonArray.length(); i++) {
+                final String name = typesJsonArray.getString(i);
+                typesResponse.put(new JSONObject() {{
+                    put(AtlasClient.NAME, name);
+                }});
+            }
+
+            JSONObject response = new JSONObject();
+            response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId());
+            response.put(AtlasClient.TYPES, typesResponse);
+            return Response.ok().entity(response).build();
+        } catch (TypeExistsException e) {
+            LOG.error("Type already exists", e);
+            throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.CONFLICT));
+        } catch (AtlasException | IllegalArgumentException e) {
+            LOG.error("Unable to persist types", e);
+            throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+        } catch (Throwable e) {
+            LOG.error("Unable to persist types", e);
+            throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+        }
+    }
+
+    /**
      * Fetch the complete definition of a given type name which is unique.
      *
      * @param typeName name of a type which is unique.

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java
index 6976c45..291ef48 100755
--- a/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java
@@ -41,7 +41,6 @@ import org.apache.atlas.typesystem.types.IDataType;
 import org.apache.atlas.typesystem.types.Multiplicity;
 import org.apache.atlas.typesystem.types.StructTypeDefinition;
 import org.apache.atlas.typesystem.types.TraitType;
-import org.apache.atlas.typesystem.types.TypeUtils;
 import org.apache.atlas.typesystem.types.utils.TypesUtil;
 import org.apache.atlas.web.util.Servlets;
 import org.apache.commons.configuration.Configuration;
@@ -188,7 +187,7 @@ public abstract class BaseResourceIT {
 
         HierarchicalTypeDefinition<TraitType> etlTraitDef = TypesUtil.createTraitTypeDef("ETL",
null);
 
-        TypesDef typesDef = TypeUtils.getTypesDef(ImmutableList.of(enumTypeDefinition),
+        TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition),
                 ImmutableList.of(structTypeDefinition),
                 ImmutableList.of(classificationTrait, piiTrait, phiTrait, pciTrait, soxTrait,
secTrait, financeTrait,
                         dimTraitDef, factTraitDef, metricTraitDef, etlTraitDef),

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
index 8053c9f..380f280 100755
--- a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
@@ -38,7 +38,6 @@ import org.apache.atlas.typesystem.types.EnumTypeDefinition;
 import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
 import org.apache.atlas.typesystem.types.StructTypeDefinition;
 import org.apache.atlas.typesystem.types.TraitType;
-import org.apache.atlas.typesystem.types.TypeUtils;
 import org.apache.atlas.typesystem.types.utils.TypesUtil;
 import org.apache.atlas.web.util.Servlets;
 import org.apache.commons.lang.RandomStringUtils;
@@ -112,6 +111,34 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
         Assert.assertNotNull(response.get(AtlasClient.GUID));
     }
 
+    @Test
+    public void testEntityDefinitionAcrossTypeUpdate() throws Exception {
+        //create type
+        HierarchicalTypeDefinition<ClassType> typeDefinition = TypesUtil
+                .createClassTypeDef(randomString(), ImmutableList.<String>of(),
+                        TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE));
+        serviceClient.createType(TypesSerialization.toJson(typeDefinition, false));
+
+        //create entity for the type
+        Referenceable instance = new Referenceable(typeDefinition.typeName);
+        instance.set("name", randomString());
+        String guid = serviceClient.createEntity(instance).getString(0);
+
+        //update type - add attribute
+        typeDefinition = TypesUtil.createClassTypeDef(typeDefinition.typeName, ImmutableList.<String>of(),
+                TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
+                TypesUtil.createOptionalAttrDef("description", DataTypes.STRING_TYPE));
+        TypesDef typeDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(),
+                ImmutableList.<StructTypeDefinition>of(), ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
+                ImmutableList.of(typeDefinition));
+        serviceClient.updateType(typeDef);
+
+        //Get definition after type update - new attributes should be null
+        Referenceable entity = serviceClient.getEntity(guid);
+        Assert.assertNull(entity.get("description"));
+        Assert.assertEquals(entity.get("name"), instance.get("name"));
+    }
+
     @DataProvider
     public Object[][] invalidAttrValues() {
         return new Object[][]{{null}, {""}};
@@ -506,10 +533,9 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
         HierarchicalTypeDefinition<ClassType> classTypeDefinition = TypesUtil
                 .createClassTypeDef(classType, ImmutableList.<String>of(),
                         TypesUtil.createUniqueRequiredAttrDef(attrName, DataTypes.STRING_TYPE));
-        TypesDef typesDef = TypeUtils
-                .getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
-                        ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
-                        ImmutableList.of(classTypeDefinition));
+        TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(),
ImmutableList.<StructTypeDefinition>of(),
+                ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
+                ImmutableList.of(classTypeDefinition));
         createType(typesDef);
 
         Referenceable instance = new Referenceable(classType);

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java
b/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java
index 84036bd..d255b75 100755
--- a/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java
@@ -32,7 +32,6 @@ import org.apache.atlas.typesystem.types.EnumTypeDefinition;
 import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
 import org.apache.atlas.typesystem.types.StructTypeDefinition;
 import org.apache.atlas.typesystem.types.TraitType;
-import org.apache.atlas.typesystem.types.TypeUtils;
 import org.apache.atlas.typesystem.types.utils.TypesUtil;
 import org.apache.atlas.web.util.Servlets;
 import org.codehaus.jettison.json.JSONArray;
@@ -176,8 +175,7 @@ public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT
{
         HierarchicalTypeDefinition<TraitType> financeTrait =
                 TypesUtil.createTraitTypeDef("Finance", ImmutableList.<String>of());
 
-        TypesDef typesDef = TypeUtils
-                .getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
+        TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(),
ImmutableList.<StructTypeDefinition>of(),
                         ImmutableList
                                 .of(classificationTraitDefinition, piiTrait, phiTrait, pciTrait,
soxTrait, secTrait,
                                         financeTrait), ImmutableList.of(dslTestTypeDefinition));

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java
b/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java
index a9e8eba..ac1b5ca 100755
--- a/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java
@@ -28,8 +28,10 @@ import org.apache.atlas.typesystem.json.TypesSerialization$;
 import org.apache.atlas.typesystem.types.AttributeDefinition;
 import org.apache.atlas.typesystem.types.ClassType;
 import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.EnumTypeDefinition;
 import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
 import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructTypeDefinition;
 import org.apache.atlas.typesystem.types.TraitType;
 import org.apache.atlas.typesystem.types.utils.TypesUtil;
 import org.apache.atlas.web.util.Servlets;
@@ -87,6 +89,33 @@ public class TypesJerseyResourceIT extends BaseResourceIT {
         }
     }
 
+    @Test
+    public void testUpdate() throws Exception {
+        HierarchicalTypeDefinition<ClassType> typeDefinition = TypesUtil
+                .createClassTypeDef(randomString(), ImmutableList.<String>of(),
+                        TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE));
+        List<String> typesCreated = serviceClient.createType(TypesSerialization.toJson(typeDefinition,
false));
+        Assert.assertEquals(typesCreated.size(), 1);
+        Assert.assertEquals(typesCreated.get(0), typeDefinition.typeName);
+
+        //Add super type
+        HierarchicalTypeDefinition<ClassType> superTypeDefinition = TypesUtil
+                .createClassTypeDef(randomString(), ImmutableList.<String>of(),
+                        TypesUtil.createOptionalAttrDef("sname", DataTypes.STRING_TYPE));
+
+        typeDefinition = TypesUtil.createClassTypeDef(typeDefinition.typeName,
+                ImmutableList.of(superTypeDefinition.typeName),
+                TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
+                TypesUtil.createOptionalAttrDef("description", DataTypes.STRING_TYPE));
+        TypesDef typeDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(),
+                ImmutableList.<StructTypeDefinition>of(), ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
+                ImmutableList.of(superTypeDefinition, typeDefinition));
+        List<String> typesUpdated = serviceClient.updateType(typeDef);
+        Assert.assertEquals(typesUpdated.size(), 2);
+        Assert.assertTrue(typesUpdated.contains(superTypeDefinition.typeName));
+        Assert.assertTrue(typesUpdated.contains(typeDefinition.typeName));
+    }
+
     @Test(dependsOnMethods = "testSubmit")
     public void testGetDefinition() throws Exception {
         for (HierarchicalTypeDefinition typeDefinition : typeDefinitions) {


Mime
View raw message