atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mad...@apache.org
Subject atlas git commit: ATLAS-2120: Import API updated to detect attribute-type change
Date Wed, 13 Sep 2017 08:46:09 GMT
Repository: atlas
Updated Branches:
  refs/heads/master 636a62137 -> 8348f2212


ATLAS-2120: Import API updated to detect attribute-type change

Signed-off-by: Madhan Neethiraj <madhan@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/8348f221
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/8348f221
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/8348f221

Branch: refs/heads/master
Commit: 8348f2212187750a20fee7bb4ebe0c1d739d6cc0
Parents: 636a621
Author: Ashutosh Mestry <amestry@hortonworks.com>
Authored: Wed Sep 13 00:00:41 2017 -0700
Committer: Madhan Neethiraj <madhan@apache.org>
Committed: Wed Sep 13 01:26:07 2017 -0700

----------------------------------------------------------------------
 docs/src/site/twiki/Import-API-Options.twiki    |   8 +--
 .../java/org/apache/atlas/AtlasErrorCode.java   |   3 +-
 .../impexp/TypeAttributeDifference.java         |  64 +++++++++++++------
 .../repository/impexp/ImportServiceTest.java    |  26 ++++++++
 .../impexp/ZipFileResourceTestUtils.java        |  27 +++++++-
 repository/src/test/resources/hdfs_path1.zip    | Bin 0 -> 2411 bytes
 repository/src/test/resources/tag1.json         |  43 +++++++++++++
 7 files changed, 143 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/atlas/blob/8348f221/docs/src/site/twiki/Import-API-Options.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/Import-API-Options.twiki b/docs/src/site/twiki/Import-API-Options.twiki
index cc09fc7..4004e70 100644
--- a/docs/src/site/twiki/Import-API-Options.twiki
+++ b/docs/src/site/twiki/Import-API-Options.twiki
@@ -68,12 +68,12 @@ To use the option, set the contents of _importOptions.json_ to:
 }
 </verbatim>
 
-To use _startIndex_, use the following in the _importOptions.json_:
+To use _startPosition_, use the following in the _importOptions.json_:
 <verbatim>
 {
 
   "options": {
-    "startIndex": "332"
+    "startPosition": "332"
   }
 }
 </verbatim>
@@ -89,7 +89,7 @@ Steps to use the behavior:
 
 The output of Export has _atlas-typedef.json_ that contains the type definitions for the
entities exported.
 
-By default (that is if no options is specified), the type definitions are imported and applied
to the system. The entity import is then performed.
+By default (that is if no options is specified), the type definitions are imported and applied
to the system being imported to. The entity import is performed after this.
 
 In some cases, you would not want to modify the type definitions. Import may be better off
failing than the types be modified.
 
@@ -100,7 +100,7 @@ Table below enumerates the conditions that get addressed as part of type
definit
 |*Condition*|*Action*|
 | Incoming type does not exist in target system | Type is created. |
 |Type to be imported and type in target system are same | No change |
-|Type to be imported and type in target system differ by some attributes| Target system type
is updated to the attributes present in the source. It is possible that the target system
will have attributes in addition to the one present in the source. In that case, the target
system's type attributes will be an union of the attributes. (Attributes in target system
will not be deleted to match the source.)|
+|Type to be imported and type in target system differ by some attributes| Target system type
is updated to the attributes present in the source. It is possible that the target system
will have attributes in addition to the one present in the source. In that case, the target
system's type attributes will be an union of the attributes. Attributes in target system will
not be deleted to match the source. If the type of the attribute differ, import process will
be aborted and exception logged.|
 
 To use the option, set the contents of _importOptions.json_ to:
 <verbatim>

http://git-wip-us.apache.org/repos/asf/atlas/blob/8348f221/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
index 2503d8e..f529739 100644
--- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
+++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
@@ -92,8 +92,9 @@ public enum AtlasErrorCode {
     RELATIONSHIP_INVALID_ENDTYPE(400, "ATLAS-400-00-045", "Invalid entity-type for relationship
attribute ‘{0}’: entity specified (guid={1}) is of type ‘{2}’, but expected type is
‘{3}’"),
     UNKNOWN_CLASSIFICATION(400, "ATLAS-400-00-046", "{0}: Unknown/invalid classification"),
     INVALID_SEARCH_PARAMS(400, "ATLAS-400-00-047", "No search parameter was found. One of
the following MUST be specified in the request; typeName, classification or queryText"),
-    INVALID_RELATIONSHIP_ATTRIBUTE(400, "ATLAS-400-00-048", "Expected attribute {0} to be
a relationship but found type {}"),
+    INVALID_RELATIONSHIP_ATTRIBUTE(400, "ATLAS-400-00-048", "Expected attribute {0} to be
a relationship but found type {1}"),
     INVALID_RELATIONSHIP_TYPE(400, "ATLAS-400-00-049", "Invalid entity type '{0}', guid '{1}'
in relationship search"),
+    INVALID_IMPORT_ATTRIBUTE_TYPE_CHANGED(400, "ATLAS-400-00-050", "Attribute {0}.{1} is
of type {2}. Import has this attribute type as {3}"),
 
     // All Not found enums go here
     TYPE_NAME_NOT_FOUND(404, "ATLAS-404-00-001", "Given typename {0} was invalid"),

http://git-wip-us.apache.org/repos/asf/atlas/blob/8348f221/repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java
index 68bc583..b3c790a 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,6 +17,7 @@
  */
 package org.apache.atlas.repository.impexp;
 
+import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.impexp.AtlasImportResult;
 import org.apache.atlas.model.typedef.AtlasClassificationDef;
@@ -26,11 +27,15 @@ import org.apache.atlas.model.typedef.AtlasStructDef;
 import org.apache.atlas.model.typedef.AtlasTypesDef;
 import org.apache.atlas.store.AtlasTypeDefStore;
 import org.apache.atlas.type.AtlasTypeRegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
 import java.util.List;
 
 public class TypeAttributeDifference {
+    private static final Logger LOG = LoggerFactory.getLogger(TypeAttributeDifference.class);
+
     private final AtlasTypeDefStore typeDefStore;
     private final AtlasTypeRegistry typeRegistry;
 
@@ -48,9 +53,9 @@ public class TypeAttributeDifference {
     }
 
     private void updateEntityDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result)
throws AtlasBaseException {
-        for (AtlasEntityDef def: typeDefinitionMap.getEntityDefs()) {
+        for (AtlasEntityDef def : typeDefinitionMap.getEntityDefs()) {
             AtlasEntityDef existing = typeRegistry.getEntityDefByName(def.getName());
-            if(existing != null && addAttributes(existing, def)) {
+            if (existing != null && addAttributes(existing, def)) {
                 typeDefStore.updateEntityDefByName(existing.getName(), existing);
                 result.incrementMeticsCounter("typedef:entitydef:update");
             }
@@ -58,9 +63,9 @@ public class TypeAttributeDifference {
     }
 
     private void updateClassificationDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult
result) throws AtlasBaseException {
-        for (AtlasClassificationDef def: typeDefinitionMap.getClassificationDefs()) {
+        for (AtlasClassificationDef def : typeDefinitionMap.getClassificationDefs()) {
             AtlasClassificationDef existing = typeRegistry.getClassificationDefByName(def.getName());
-            if(existing != null && addAttributes(existing, def)) {
+            if (existing != null && addAttributes(existing, def)) {
                 typeDefStore.updateClassificationDefByName(existing.getName(), existing);
                 result.incrementMeticsCounter("typedef:classification:update");
             }
@@ -68,9 +73,9 @@ public class TypeAttributeDifference {
     }
 
     private void updateEnumDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result)
throws AtlasBaseException {
-        for (AtlasEnumDef def: typeDefinitionMap.getEnumDefs()) {
+        for (AtlasEnumDef def : typeDefinitionMap.getEnumDefs()) {
             AtlasEnumDef existing = typeRegistry.getEnumDefByName(def.getName());
-            if(existing != null && addElements(existing, def)) {
+            if (existing != null && addElements(existing, def)) {
                 typeDefStore.updateEnumDefByName(existing.getName(), existing);
                 result.incrementMeticsCounter("typedef:enum:update");
             }
@@ -78,45 +83,62 @@ public class TypeAttributeDifference {
     }
 
     private void updateStructDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result)
throws AtlasBaseException {
-        for (AtlasStructDef def: typeDefinitionMap.getStructDefs()) {
+        for (AtlasStructDef def : typeDefinitionMap.getStructDefs()) {
             AtlasStructDef existing = typeRegistry.getStructDefByName(def.getName());
-            if(existing != null && addAttributes(existing, def)) {
+            if (existing != null && addAttributes(existing, def)) {
                 typeDefStore.updateStructDefByName(existing.getName(), existing);
                 result.incrementMeticsCounter("typedef:struct:update");
             }
         }
     }
 
-    private boolean addElements(AtlasEnumDef existing, AtlasEnumDef incoming) {
+    private boolean addElements(AtlasEnumDef existing, AtlasEnumDef incoming) throws AtlasBaseException
{
         return addElements(existing, getElementsAbsentInExisting(existing, incoming));
     }
 
-    private boolean addAttributes(AtlasStructDef existing, AtlasStructDef incoming) {
+    private boolean addAttributes(AtlasStructDef existing, AtlasStructDef incoming) throws
AtlasBaseException {
         return addAttributes(existing, getElementsAbsentInExisting(existing, incoming));
     }
 
-    private List<AtlasStructDef.AtlasAttributeDef> getElementsAbsentInExisting(AtlasStructDef
existing, AtlasStructDef incoming) {
+    private List<AtlasStructDef.AtlasAttributeDef> getElementsAbsentInExisting(AtlasStructDef
existing, AtlasStructDef incoming) throws AtlasBaseException {
         List<AtlasStructDef.AtlasAttributeDef> difference = new ArrayList<>();
         for (AtlasStructDef.AtlasAttributeDef attr : incoming.getAttributeDefs()) {
-            if(existing.getAttribute(attr.getName()) == null) {
-                difference.add(attr);
-            }
+            updateCollectionWithDifferingAttributes(difference, existing, attr);
         }
 
         return difference;
     }
 
-    private List<AtlasEnumDef.AtlasEnumElementDef> getElementsAbsentInExisting(AtlasEnumDef
existing, AtlasEnumDef incoming) {
+    private void updateCollectionWithDifferingAttributes(List<AtlasStructDef.AtlasAttributeDef>
difference,
+                                                         AtlasStructDef existing,
+                                                         AtlasStructDef.AtlasAttributeDef
incoming) throws AtlasBaseException {
+        AtlasStructDef.AtlasAttributeDef existingAttribute = existing.getAttribute(incoming.getName());
+        if (existingAttribute == null) {
+            difference.add(incoming);
+        } else {
+            if (!existingAttribute.getTypeName().equals(incoming.getTypeName())) {
+                LOG.error("Attribute definition difference found: {}, {}", existingAttribute,
incoming);
+                throw new AtlasBaseException(AtlasErrorCode.INVALID_IMPORT_ATTRIBUTE_TYPE_CHANGED,
existing.getName(), existingAttribute.getName(), existingAttribute.getTypeName(), incoming.getTypeName());
+            }
+        }
+    }
+
+    private List<AtlasEnumDef.AtlasEnumElementDef> getElementsAbsentInExisting(AtlasEnumDef
existing, AtlasEnumDef incoming) throws AtlasBaseException {
         List<AtlasEnumDef.AtlasEnumElementDef> difference = new ArrayList<>();
         for (AtlasEnumDef.AtlasEnumElementDef ed : incoming.getElementDefs()) {
-            if(existing.getElement(ed.getValue()) == null) {
-                difference.add(ed);
-            }
+            updateCollectionWithDifferingAttributes(existing, difference, ed);
         }
 
         return difference;
     }
 
+    private void updateCollectionWithDifferingAttributes(AtlasEnumDef existing, List<AtlasEnumDef.AtlasEnumElementDef>
difference, AtlasEnumDef.AtlasEnumElementDef ed) throws AtlasBaseException {
+        AtlasEnumDef.AtlasEnumElementDef existingElement = existing.getElement(ed.getValue());
+        if (existingElement == null) {
+            difference.add(ed);
+        }
+    }
+
     private boolean addAttributes(AtlasStructDef def, List<AtlasStructDef.AtlasAttributeDef>
list) {
         for (AtlasStructDef.AtlasAttributeDef ad : list) {
             def.addAttribute(ad);

http://git-wip-us.apache.org/repos/asf/atlas/blob/8348f221/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
index 17813d6..3359850 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
@@ -18,6 +18,7 @@
 package org.apache.atlas.repository.impexp;
 
 import com.google.inject.Inject;
+import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.RequestContextV1;
 import org.apache.atlas.TestModules;
 import org.apache.atlas.TestUtilsV2;
@@ -25,6 +26,7 @@ import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.impexp.AtlasImportRequest;
 import org.apache.atlas.repository.store.graph.AtlasEntityStore;
 import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasClassificationType;
 import org.apache.atlas.type.AtlasTypeRegistry;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -41,6 +43,7 @@ import java.util.Map;
 import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.*;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
 
 @Guice(modules = TestModules.TestOnlyModule.class)
 public class ImportServiceTest {
@@ -156,6 +159,29 @@ public class ImportServiceTest {
         runImportWithNoParameters(getImportService(), zipSource);
     }
 
+    @DataProvider(name = "hdfs_path1")
+    public static Object[][] getDataFromHdfsPath1(ITestContext context) throws IOException
{
+        return getZipSource("hdfs_path1.zip");
+    }
+
+
+    @Test(dataProvider = "hdfs_path1", expectedExceptions = AtlasBaseException.class)
+    public void importHdfs_path1(ZipSource zipSource) throws IOException, AtlasBaseException
{
+        loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
+        loadModelFromJson("1000-Hadoop/1020-fs_model.json", typeDefStore, typeRegistry);
+        loadModelFromResourcesJson("tag1.json", typeDefStore, typeRegistry);
+
+        try {
+            runImportWithNoParameters(getImportService(), zipSource);
+        } catch (AtlasBaseException e) {
+            assertEquals(e.getAtlasErrorCode(), AtlasErrorCode.INVALID_IMPORT_ATTRIBUTE_TYPE_CHANGED);
+            AtlasClassificationType tag1 = typeRegistry.getClassificationTypeByName("tag1");
+            assertNotNull(tag1);
+            assertEquals(tag1.getAllAttributes().size(), 2);
+            throw e;
+        }
+    }
+
     private ImportService getImportService() {
         return new ImportService(typeDefStore, entityStore, typeRegistry);
     }

http://git-wip-us.apache.org/repos/asf/atlas/blob/8348f221/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
index d80fab0..f0dab47 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
@@ -49,8 +49,7 @@ public class ZipFileResourceTestUtils {
     public static final Logger LOG = LoggerFactory.getLogger(ZipFileResourceTestUtils.class);
 
     public static FileInputStream getFileInputStream(String fileName) {
-        final String userDir = System.getProperty("user.dir");
-        String filePath = getFilePath(userDir, fileName);
+        String filePath = getFileFromResources(fileName);
         File f = new File(filePath);
         FileInputStream fs = null;
         try {
@@ -61,6 +60,11 @@ public class ZipFileResourceTestUtils {
         return fs;
     }
 
+    private static String getFileFromResources(String fileName) {
+        final String userDir = System.getProperty("user.dir");
+        return getFilePath(userDir, fileName);
+    }
+
     private static String getFilePath(String startPath, String fileName) {
         return startPath + "/src/test/resources/" + fileName;
     }
@@ -75,6 +79,15 @@ public class ZipFileResourceTestUtils {
         return s;
     }
 
+    public static String getModelJsonFromResources(String fileName) throws IOException {
+        String filePath = getFileFromResources(fileName);
+        File f = new File(filePath);
+        String s = FileUtils.readFileToString(f);
+        assertFalse(StringUtils.isEmpty(s), "Model file read correctly from resources!");
+
+        return s;
+    }
+
     public static Object[][] getZipSource(String fileName) throws IOException {
         FileInputStream fs = ZipFileResourceTestUtils.getFileInputStream(fileName);
 
@@ -119,6 +132,11 @@ public class ZipFileResourceTestUtils {
         createTypesAsNeeded(typesFromJson, typeDefStore, typeRegistry);
     }
 
+    public static void loadModelFromResourcesJson(String fileName, AtlasTypeDefStore typeDefStore,
AtlasTypeRegistry typeRegistry) throws IOException, AtlasBaseException {
+        AtlasTypesDef typesFromJson = getAtlasTypesDefFromResourceFile(fileName);
+        createTypesAsNeeded(typesFromJson, typeDefStore, typeRegistry);
+    }
+
     private static void createTypesAsNeeded(AtlasTypesDef typesFromJson, AtlasTypeDefStore
typeDefStore, AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
         AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typesFromJson,
typeRegistry);
 
@@ -132,6 +150,11 @@ public class ZipFileResourceTestUtils {
         return AtlasType.fromJson(sampleTypes, AtlasTypesDef.class);
     }
 
+    private static AtlasTypesDef getAtlasTypesDefFromResourceFile(String fileName) throws
IOException {
+        String sampleTypes = getModelJsonFromResources(fileName);
+        return AtlasType.fromJson(sampleTypes, AtlasTypesDef.class);
+    }
+
     public static AtlasImportRequest getDefaultImportRequest() {
         return new AtlasImportRequest();
     }

http://git-wip-us.apache.org/repos/asf/atlas/blob/8348f221/repository/src/test/resources/hdfs_path1.zip
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/hdfs_path1.zip b/repository/src/test/resources/hdfs_path1.zip
new file mode 100644
index 0000000..1f4449d
Binary files /dev/null and b/repository/src/test/resources/hdfs_path1.zip differ

http://git-wip-us.apache.org/repos/asf/atlas/blob/8348f221/repository/src/test/resources/tag1.json
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/tag1.json b/repository/src/test/resources/tag1.json
new file mode 100644
index 0000000..c49ae65
--- /dev/null
+++ b/repository/src/test/resources/tag1.json
@@ -0,0 +1,43 @@
+{
+  "enumDefs": [],
+  "structDefs": [],
+  "classificationDefs": [
+    {
+      "category": "CLASSIFICATION",
+      "guid": "03f95af3-3415-4522-8f4c-95fadcb59636",
+      "createdBy": "admin",
+      "updatedBy": "admin",
+      "createTime": 1504887556449,
+      "updateTime": 1504887556449,
+      "version": 1,
+      "name": "tag1",
+      "description": "tag1",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "attrib1",
+          "typeName": "date",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "attrib3",
+          "typeName": "int",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": []
+    }
+  ],
+  "entityDefs": [],
+  "relationshipDefs": []
+}


Mime
View raw message