atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mad...@apache.org
Subject incubator-atlas git commit: ATLAS-1724, ATLAS-1722: fix export to report error while exporting a non-existing entity
Date Tue, 02 May 2017 00:12:25 GMT
Repository: incubator-atlas
Updated Branches:
  refs/heads/0.8-incubating 2fd12efc4 -> ef6b1c1ed


ATLAS-1724, ATLAS-1722: fix export to report error while exporting a non-existing entity

Signed-off-by: Madhan Neethiraj <madhan@apache.org>
(cherry picked from commit 09425abc7ca92c5ec84480c4d39451fa175f2f8c)


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/ef6b1c1e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/ef6b1c1e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/ef6b1c1e

Branch: refs/heads/0.8-incubating
Commit: ef6b1c1ed821081764fd6e2513da0c984e127d26
Parents: 2fd12ef
Author: ashutoshm <amestry@hortonworks.com>
Authored: Mon May 1 10:22:33 2017 -0700
Committer: Madhan Neethiraj <madhan@apache.org>
Committed: Mon May 1 17:12:21 2017 -0700

----------------------------------------------------------------------
 repository/pom.xml                              |   6 +
 .../atlas/repository/impexp/ExportService.java  | 114 ++++++++++++------
 .../repository/impexp/ExportServiceTest.java    | 116 ++++++++++++++-----
 3 files changed, 168 insertions(+), 68 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ef6b1c1e/repository/pom.xml
----------------------------------------------------------------------
diff --git a/repository/pom.xml b/repository/pom.xml
index 10edda4..415bccb 100755
--- a/repository/pom.xml
+++ b/repository/pom.xml
@@ -33,6 +33,12 @@
 
     <dependencies>
         <dependency>
+            <groupId>org.powermock</groupId>
+            <artifactId>powermock-reflect</artifactId>
+            <version>1.6.1</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
             <groupId>org.apache.atlas</groupId>
             <artifactId>atlas-intg</artifactId>
         </dependency>

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ef6b1c1e/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
index 73c3140..2f298ba 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
@@ -92,64 +92,101 @@ public class ExportService {
         try {
             LOG.info("==> export(user={}, from={})", userName, requestingIP);
 
-            for (AtlasObjectId item : request.getItemsToExport()) {
-                processObjectId(item, context);
-            }
+            AtlasExportResult.OperationStatus[] statuses = processItems(request, context);
 
-            long endTime = System.currentTimeMillis();
+            processTypesDef(context);
 
-            AtlasTypesDef typesDef = context.result.getData().getTypesDef();
+            updateSinkWithOperationMetrics(context, statuses, getOperationDuration(startTime));
+        } catch(Exception ex) {
+            LOG.error("Operation failed: ", ex);
+        } finally {
+            atlasGraph.releaseGremlinScriptEngine(context.scriptEngine);
+            LOG.info("<== export(user={}, from={}): status {}", userName, requestingIP,
context.result.getOperationStatus());
+            context.clear();
+            result.clear();
+        }
 
-            for (String entityType : context.entityTypes) {
-                AtlasEntityDef entityDef = typeRegistry.getEntityDefByName(entityType);
+        return context.result;
+    }
 
-                typesDef.getEntityDefs().add(entityDef);
-            }
+    private void updateSinkWithOperationMetrics(ExportContext context, AtlasExportResult.OperationStatus[]
statuses, int duration) throws AtlasBaseException {
+        context.sink.setExportOrder(context.result.getData().getEntityCreationOrder());
+        context.sink.setTypesDef(context.result.getData().getTypesDef());
+        clearContextData(context);
+        context.result.setOperationStatus(getOverallOperationStatus(statuses));
+        context.result.incrementMeticsCounter("duration", duration);
+        context.sink.setResult(context.result);
+    }
 
-            for (String classificationType : context.classificationTypes) {
-                AtlasClassificationDef classificationDef = typeRegistry.getClassificationDefByName(classificationType);
+    private void clearContextData(ExportContext context) {
+        context.result.setData(null);
+    }
 
-                typesDef.getClassificationDefs().add(classificationDef);
-            }
+    private int getOperationDuration(long startTime) {
+        return (int) (System.currentTimeMillis() - startTime);
+    }
 
-            for (String structType : context.structTypes) {
-                AtlasStructDef structDef = typeRegistry.getStructDefByName(structType);
+    private void processTypesDef(ExportContext context) {
+        AtlasTypesDef typesDef = context.result.getData().getTypesDef();
 
-                typesDef.getStructDefs().add(structDef);
-            }
+        for (String entityType : context.entityTypes) {
+            AtlasEntityDef entityDef = typeRegistry.getEntityDefByName(entityType);
 
-            for (String enumType : context.enumTypes) {
-                AtlasEnumDef enumDef = typeRegistry.getEnumDefByName(enumType);
+            typesDef.getEntityDefs().add(entityDef);
+        }
 
-                typesDef.getEnumDefs().add(enumDef);
-            }
+        for (String classificationType : context.classificationTypes) {
+            AtlasClassificationDef classificationDef = typeRegistry.getClassificationDefByName(classificationType);
 
-            context.sink.setExportOrder(context.result.getData().getEntityCreationOrder());
-            context.sink.setTypesDef(context.result.getData().getTypesDef());
-            context.result.setData(null);
-            context.result.setOperationStatus(AtlasExportResult.OperationStatus.SUCCESS);
-            context.result.incrementMeticsCounter("duration", (int) (endTime - startTime));
+            typesDef.getClassificationDefs().add(classificationDef);
+        }
 
-            context.sink.setResult(context.result);
-        } catch(Exception ex) {
-            LOG.error("Operation failed: ", ex);
-        } finally {
-            atlasGraph.releaseGremlinScriptEngine(context.scriptEngine);
-            LOG.info("<== export(user={}, from={}): status {}", userName, requestingIP,
context.result.getOperationStatus());
-            context.clear();
-            result.clear();
+        for (String structType : context.structTypes) {
+            AtlasStructDef structDef = typeRegistry.getStructDefByName(structType);
+
+            typesDef.getStructDefs().add(structDef);
         }
 
-        return context.result;
+        for (String enumType : context.enumTypes) {
+            AtlasEnumDef enumDef = typeRegistry.getEnumDefByName(enumType);
+
+            typesDef.getEnumDefs().add(enumDef);
+        }
     }
 
-    private void processObjectId(AtlasObjectId item, ExportContext context) throws AtlasServiceException,
AtlasException, AtlasBaseException {
+    private AtlasExportResult.OperationStatus[] processItems(AtlasExportRequest request,
ExportContext context) throws AtlasServiceException, AtlasException, AtlasBaseException {
+        AtlasExportResult.OperationStatus statuses[] = new AtlasExportResult.OperationStatus[request.getItemsToExport().size()];
+        List<AtlasObjectId> itemsToExport = request.getItemsToExport();
+        for (int i = 0; i < itemsToExport.size(); i++) {
+            AtlasObjectId item = itemsToExport.get(i);
+            statuses[i] = processObjectId(item, context);
+        }
+        return statuses;
+    }
+
+    private AtlasExportResult.OperationStatus getOverallOperationStatus(AtlasExportResult.OperationStatus...
statuses) {
+        AtlasExportResult.OperationStatus overall = (statuses.length == 0) ?
+                AtlasExportResult.OperationStatus.FAIL : statuses[0];
+
+        for (AtlasExportResult.OperationStatus s : statuses) {
+            if (overall != s) {
+                overall = AtlasExportResult.OperationStatus.PARTIAL_SUCCESS;
+            }
+        }
+
+        return overall;
+    }
+
+    private AtlasExportResult.OperationStatus processObjectId(AtlasObjectId item, ExportContext
context) throws AtlasServiceException, AtlasException, AtlasBaseException {
         if (LOG.isDebugEnabled()) {
             LOG.debug("==> processObjectId({})", item);
         }
 
         try {
             List<AtlasEntityWithExtInfo> entities = getStartingEntity(item, context);
+            if(entities.size() == 0) {
+                return AtlasExportResult.OperationStatus.FAIL;
+            }
 
             for (AtlasEntityWithExtInfo entityWithExtInfo : entities) {
                 processEntity(entityWithExtInfo.getEntity().getGuid(), context);
@@ -167,14 +204,15 @@ public class ExportService {
                 }
             }
         } catch (AtlasBaseException excp) {
-            context.result.setOperationStatus(AtlasExportResult.OperationStatus.PARTIAL_SUCCESS);
-
             LOG.error("Fetching entity failed for: {}", item, excp);
+            return AtlasExportResult.OperationStatus.FAIL;
         }
 
         if (LOG.isDebugEnabled()) {
             LOG.debug("<== processObjectId({})", item);
         }
+
+        return AtlasExportResult.OperationStatus.SUCCESS;
     }
 
     private List<AtlasEntityWithExtInfo> getStartingEntity(AtlasObjectId item, ExportContext
context) throws AtlasBaseException {

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ef6b1c1e/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
index a7c70a9..a7fc24c 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
@@ -35,7 +35,8 @@ import org.apache.atlas.repository.store.graph.v1.DeleteHandlerV1;
 import org.apache.atlas.repository.store.graph.v1.SoftDeleteHandlerV1;
 import org.apache.atlas.store.AtlasTypeDefStore;
 import org.apache.atlas.type.AtlasTypeRegistry;
-import org.junit.Assert;
+import org.testng.Assert;
+import org.powermock.reflect.Whitebox;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testng.annotations.BeforeClass;
@@ -53,6 +54,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static org.testng.Assert.*;
 import static org.mockito.Mockito.mock;
 
 @Guice(modules = RepositoryMetadataModule.class)
@@ -158,7 +160,6 @@ public class ExportServiceTest {
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         ZipSink zipSink = new ZipSink(baos);
         AtlasExportResult result = exportService.run(zipSink, request, userName, hostName,
requestingIP);
-        Assert.assertEquals(result.getOperationStatus(), AtlasExportResult.OperationStatus.SUCCESS);
 
         zipSink.close();
 
@@ -177,10 +178,10 @@ public class ExportServiceTest {
         ZipSink zipSink = new ZipSink(baos);
         AtlasExportResult result = exportService.run(zipSink, request, "admin", hostName,
requestingIP);
 
-        Assert.assertNotNull(exportService);
-        Assert.assertEquals(result.getHostName(), hostName);
-        Assert.assertEquals(result.getClientIpAddress(), requestingIP);
-        Assert.assertEquals(request, result.getRequest());
+        assertNotNull(exportService);
+        assertEquals(result.getHostName(), hostName);
+        assertEquals(result.getClientIpAddress(), requestingIP);
+        assertEquals(request, result.getRequest());
     }
 
     @Test
@@ -198,8 +199,8 @@ public class ExportServiceTest {
         ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
         ZipSource zipSource = new ZipSource(bais);
 
-        Assert.assertNotNull(exportService);
-        Assert.assertNotNull(zipSource.getCreationOrder());
+        assertNotNull(exportService);
+        assertNotNull(zipSource.getCreationOrder());
         Assert.assertFalse(zipSource.hasNext());
     }
 
@@ -244,54 +245,109 @@ public class ExportServiceTest {
         verifyExportForEmployeeData(zipSource);
     }
 
+    @Test
+    public void verifyOverallStatus() throws Exception {
+
+        ExportService service = new ExportService(typeRegistry);
+        assertEquals(AtlasExportResult.OperationStatus.FAIL, Whitebox.invokeMethod(service,
+                "getOverallOperationStatus"));
+
+        assertEquals(AtlasExportResult.OperationStatus.SUCCESS, Whitebox.invokeMethod(service,
+                "getOverallOperationStatus",
+                AtlasExportResult.OperationStatus.SUCCESS));
+
+        assertEquals(AtlasExportResult.OperationStatus.SUCCESS, Whitebox.invokeMethod(service,
+                "getOverallOperationStatus",
+                                AtlasExportResult.OperationStatus.SUCCESS,
+                                AtlasExportResult.OperationStatus.SUCCESS,
+                                AtlasExportResult.OperationStatus.SUCCESS));
+
+        assertEquals(AtlasExportResult.OperationStatus.PARTIAL_SUCCESS, Whitebox.invokeMethod(service,
+                "getOverallOperationStatus",
+                AtlasExportResult.OperationStatus.FAIL,
+                AtlasExportResult.OperationStatus.PARTIAL_SUCCESS,
+                AtlasExportResult.OperationStatus.SUCCESS));
+
+        assertEquals(AtlasExportResult.OperationStatus.PARTIAL_SUCCESS, Whitebox.invokeMethod(service,
+                "getOverallOperationStatus",
+                AtlasExportResult.OperationStatus.FAIL,
+                AtlasExportResult.OperationStatus.FAIL,
+                AtlasExportResult.OperationStatus.PARTIAL_SUCCESS));
+
+        assertEquals(AtlasExportResult.OperationStatus.FAIL, Whitebox.invokeMethod(service,
+                "getOverallOperationStatus",
+                AtlasExportResult.OperationStatus.FAIL,
+                AtlasExportResult.OperationStatus.FAIL,
+                AtlasExportResult.OperationStatus.FAIL));
+
+
+    }
+
+    @Test
+    public void requestingExportOfNonExistentEntity_ReturnsFailure() throws Exception {
+        AtlasExportRequest request = getRequestForEmployee();
+        tamperEmployeeRequest(request);
+        ZipSource zipSource = runExportWithParameters(request);
+
+        assertNotNull(zipSource.getCreationOrder());
+        assertEquals(zipSource.getCreationOrder().size(), 0);
+        assertEquals(AtlasExportResult.OperationStatus.FAIL, zipSource.getExportResult().getOperationStatus());
+    }
+
+    private void tamperEmployeeRequest(AtlasExportRequest request) {
+        AtlasObjectId objectId = request.getItemsToExport().get(0);
+        objectId.getUniqueAttributes().remove("name");
+        objectId.getUniqueAttributes().put("qualifiedName", "XXX@121");
+    }
+
     private void verifyExportForEmployeeData(ZipSource zipSource) throws AtlasBaseException
{
         final List<String> expectedEntityTypes = Arrays.asList(new String[]{"Manager",
"Employee", "Department"});
 
-        Assert.assertNotNull(zipSource.getCreationOrder());
-        Assert.assertEquals(zipSource.getCreationOrder().size(), 2);
-        Assert.assertTrue(zipSource.hasNext());
+        assertNotNull(zipSource.getCreationOrder());
+        assertEquals(zipSource.getCreationOrder().size(), 2);
+        assertTrue(zipSource.hasNext());
 
         while (zipSource.hasNext()) {
             AtlasEntity entity = zipSource.next();
 
-            Assert.assertNotNull(entity);
-            Assert.assertEquals(entity.getStatus(), AtlasEntity.Status.ACTIVE);
-            Assert.assertTrue(expectedEntityTypes.contains(entity.getTypeName()));
+            assertNotNull(entity);
+            assertEquals(AtlasEntity.Status.ACTIVE, entity.getStatus());
+            assertTrue(expectedEntityTypes.contains(entity.getTypeName()));
         }
 
         verifyTypeDefs(zipSource);
     }
 
     private void verifyExportForHrData(ZipSource zipSource) throws IOException, AtlasBaseException
{
-        Assert.assertNotNull(zipSource.getCreationOrder());
-        Assert.assertTrue(zipSource.getCreationOrder().size() == 1);
-        Assert.assertTrue(zipSource.hasNext());
+        assertNotNull(zipSource.getCreationOrder());
+        assertTrue(zipSource.getCreationOrder().size() == 1);
+        assertTrue(zipSource.hasNext());
 
         AtlasEntity entity = zipSource.next();
 
-        Assert.assertNotNull(entity);
-        Assert.assertTrue(entity.getTypeName().equals("Department"));
-        Assert.assertEquals(entity.getStatus(), AtlasEntity.Status.ACTIVE);
+        assertNotNull(entity);
+        assertTrue(entity.getTypeName().equals("Department"));
+        assertEquals(entity.getStatus(), AtlasEntity.Status.ACTIVE);
         verifyTypeDefs(zipSource);
     }
 
     private void verifyExportForHrDataForConnected(ZipSource zipSource) throws IOException,
AtlasBaseException {
-        Assert.assertNotNull(zipSource.getCreationOrder());
-        Assert.assertTrue(zipSource.getCreationOrder().size() == 2);
-        Assert.assertTrue(zipSource.hasNext());
+        assertNotNull(zipSource.getCreationOrder());
+        assertTrue(zipSource.getCreationOrder().size() == 2);
+        assertTrue(zipSource.hasNext());
 
         AtlasEntity entity = zipSource.next();
 
-        Assert.assertNotNull(entity);
-        Assert.assertTrue(entity.getTypeName().equals("Department"));
-        Assert.assertEquals(entity.getStatus(), AtlasEntity.Status.ACTIVE);
+        assertNotNull(entity);
+        assertTrue(entity.getTypeName().equals("Department"));
+        assertEquals(entity.getStatus(), AtlasEntity.Status.ACTIVE);
         verifyTypeDefs(zipSource);
     }
 
     private void verifyTypeDefs(ZipSource zipSource) throws AtlasBaseException {
-        Assert.assertEquals(zipSource.getTypesDef().getEnumDefs().size(), 1);
-        Assert.assertEquals(zipSource.getTypesDef().getClassificationDefs().size(), 0);
-        Assert.assertEquals(zipSource.getTypesDef().getStructDefs().size(), 1);
-        Assert.assertEquals(zipSource.getTypesDef().getEntityDefs().size(), 4);
+        assertEquals(zipSource.getTypesDef().getEnumDefs().size(), 1);
+        assertEquals(zipSource.getTypesDef().getClassificationDefs().size(), 0);
+        assertEquals(zipSource.getTypesDef().getStructDefs().size(), 1);
+        assertEquals(zipSource.getTypesDef().getEntityDefs().size(), 4);
     }
 }


Mime
View raw message