atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shweth...@apache.org
Subject incubator-atlas git commit: ATLAS-738 Add query ability on system properties like guid, state, createdtime etc (shwethags)
Date Thu, 12 May 2016 06:32:51 GMT
Repository: incubator-atlas
Updated Branches:
  refs/heads/master 915595780 -> bfd5f5caa


ATLAS-738 Add query ability on system properties like guid, state, createdtime etc (shwethags)


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/bfd5f5ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/bfd5f5ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/bfd5f5ca

Branch: refs/heads/master
Commit: bfd5f5caa609b9084ec816deb4c6bf11eec18bab
Parents: 9155957
Author: Shwetha GS <sshivalingamurthy@hortonworks.com>
Authored: Thu May 12 12:02:46 2016 +0530
Committer: Shwetha GS <sshivalingamurthy@hortonworks.com>
Committed: Thu May 12 12:02:46 2016 +0530

----------------------------------------------------------------------
 release-log.txt                                 |  1 +
 .../org/apache/atlas/repository/Constants.java  | 16 ++++
 .../graph/GraphBackedMetadataRepository.java    |  3 +
 .../org/apache/atlas/query/TypeUtils.scala      |  6 ++
 .../GraphBackedDiscoveryServiceTest.java        | 98 +++++++++++++-------
 .../atlas/typesystem/types/AttributeInfo.java   |  2 +-
 .../atlas/typesystem/types/utils/TypesUtil.java | 14 +++
 7 files changed, 106 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bfd5f5ca/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 7e3149c..66064d6 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -20,6 +20,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file
(dosset
 ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via
shwethags)
 
 ALL CHANGES:
+ATLAS-738 Add query ability on system properties like guid, state, createdtime etc (shwethags)
 ATLAS-692 Create abstraction layer for graph databases (jnhagelb via yhemanth)
 ATLAS-689 Migrate Atlas-Storm integration to use Storm 1.0 dependencies. (svimal2106 via
yhemanth)
 ATLAS-754 InstanceSerialization does not serialize Reference in the values array of Reference.(harishjp
via sumasai)

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bfd5f5ca/repository/src/main/java/org/apache/atlas/repository/Constants.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/Constants.java b/repository/src/main/java/org/apache/atlas/repository/Constants.java
index 0832c81..893f1b6 100755
--- a/repository/src/main/java/org/apache/atlas/repository/Constants.java
+++ b/repository/src/main/java/org/apache/atlas/repository/Constants.java
@@ -18,6 +18,10 @@
 
 package org.apache.atlas.repository;
 
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+
 public final class Constants {
 
     /**
@@ -62,6 +66,18 @@ public final class Constants {
     public static final String TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "timestamp";
     public static final String MODIFICATION_TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX
+ "modificationTimestamp";
 
+    public static AttributeInfo getAttributeInfoForSystemAttributes(String field) {
+        switch (field) {
+        case STATE_PROPERTY_KEY:
+        case GUID_PROPERTY_KEY:
+            return TypesUtil.newAttributeInfo(field, DataTypes.STRING_TYPE);
+
+        case TIMESTAMP_PROPERTY_KEY:
+        case MODIFICATION_TIMESTAMP_PROPERTY_KEY:
+            return TypesUtil.newAttributeInfo(field, DataTypes.LONG_TYPE);
+        }
+        return null;
+    }
 
     /**
      * search backing index name.

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bfd5f5ca/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
index de29e86..3604277 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
@@ -105,6 +105,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository
{
 
     @Override
     public String getFieldNameInVertex(IDataType<?> dataType, AttributeInfo aInfo)
throws AtlasException {
+        if (aInfo.name.startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX)) {
+            return aInfo.name;
+        }
         return GraphHelper.getQualifiedFieldName(dataType, aInfo.name);
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bfd5f5ca/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala b/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala
index 552cc32..5a64c53 100755
--- a/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala
+++ b/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala
@@ -23,6 +23,7 @@ import java.util.concurrent.atomic.AtomicInteger
 
 import org.apache.atlas.AtlasException
 import org.apache.atlas.query.Expressions.{PathExpression, SelectExpression}
+import org.apache.atlas.repository.Constants
 import org.apache.atlas.typesystem.types.DataTypes.{ArrayType, PrimitiveType, TypeCategory}
 import org.apache.atlas.typesystem.types._
 
@@ -203,6 +204,11 @@ object TypeUtils {
                 return Some(FieldInfo(typ,fMap.get.fields.get(id)))
             }
 
+            val systemField = Constants.getAttributeInfoForSystemAttributes(id)
+            if (systemField != null) {
+              return Some(FieldInfo(systemField.dataType(), systemField))
+            }
+
             try {
               val FIELD_QUALIFIER(clsNm, rest) = id
                 val idTyp = typSystem.getDataType(classOf[IDataType[_]], clsNm)

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bfd5f5ca/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
index 9cc501f..3b50dfb 100755
--- a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
@@ -56,6 +56,8 @@ import java.util.Map;
 import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
 import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
 import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
 
 @Guice(modules = RepositoryMetadataModule.class)
 public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
@@ -94,32 +96,62 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest
{
     }
 
     @Test
+    public void testSearchBySystemProperties() throws Exception {
+        //system property in select
+        String dslQuery = "from Department select __guid";
+
+        String jsonResults = discoveryService.searchByDSL(dslQuery);
+        assertNotNull(jsonResults);
+
+        JSONObject results = new JSONObject(jsonResults);
+        assertEquals(results.length(), 3);
+
+        JSONArray rows = results.getJSONArray("rows");
+        assertNotNull(rows);
+        assertEquals(rows.length(), 1);
+        assertNotNull(rows.getJSONObject(0).getString("__guid"));
+
+        //system property in where clause
+        String guid = rows.getJSONObject(0).getString("__guid");
+        dslQuery = "Department where __guid = '" + guid + "' and __state = 'ACTIVE'";
+        jsonResults = discoveryService.searchByDSL(dslQuery);
+        assertNotNull(jsonResults);
+
+        results = new JSONObject(jsonResults);
+        assertEquals(results.length(), 3);
+
+        rows = results.getJSONArray("rows");
+        assertNotNull(rows);
+        assertEquals(rows.length(), 1);
+    }
+
+    @Test
     public void testSearchByDSLReturnsEntity() throws Exception {
         String dslQuery = "from Department";
 
         String jsonResults = discoveryService.searchByDSL(dslQuery);
-        Assert.assertNotNull(jsonResults);
+        assertNotNull(jsonResults);
 
         JSONObject results = new JSONObject(jsonResults);
-        Assert.assertEquals(results.length(), 3);
+        assertEquals(results.length(), 3);
         System.out.println("results = " + results);
 
         Object query = results.get("query");
-        Assert.assertNotNull(query);
+        assertNotNull(query);
 
         JSONObject dataType = results.getJSONObject("dataType");
-        Assert.assertNotNull(dataType);
+        assertNotNull(dataType);
         String typeName = dataType.getString("typeName");
-        Assert.assertNotNull(typeName);
-        Assert.assertEquals(typeName, "Department");
+        assertNotNull(typeName);
+        assertEquals(typeName, "Department");
 
         JSONArray rows = results.getJSONArray("rows");
-        Assert.assertNotNull(rows);
-        Assert.assertEquals(rows.length(), 1);
+        assertNotNull(rows);
+        assertEquals(rows.length(), 1);
 
         //Assert that entity state is set in the result entities
         String entityState = rows.getJSONObject(0).getJSONObject("$id$").getString("state");
-        Assert.assertEquals(entityState, Id.EntityState.ACTIVE.name());
+        assertEquals(entityState, Id.EntityState.ACTIVE.name());
     }
 
     @Test(expectedExceptions = Throwable.class)
@@ -167,11 +199,11 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest
{
         Assert.assertTrue(resultList.size() > 0);
         for (Map<String, Object> vertexProps : resultList) {
             Object object = vertexProps.get(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
-            Assert.assertNotNull(object);
+            assertNotNull(object);
             Long timestampAsLong = Long.valueOf((String)object);
             Assert.assertTrue(timestampAsLong > 1420070400000L);
             object = vertexProps.get(Constants.TIMESTAMP_PROPERTY_KEY);
-            Assert.assertNotNull(object);
+            assertNotNull(object);
         }
     }
 
@@ -490,23 +522,23 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest
{
     public void  testSearchByDSLQueriesWithOrderBy(String dslQuery, Integer expectedNumRows,
String orderBy, boolean ascending) throws Exception {
         System.out.println("Executing dslQuery = " + dslQuery);
         String jsonResults = discoveryService.searchByDSL(dslQuery);
-        Assert.assertNotNull(jsonResults);
+        assertNotNull(jsonResults);
 
         JSONObject results = new JSONObject(jsonResults);
-        Assert.assertEquals(results.length(), 3);
+        assertEquals(results.length(), 3);
 
         Object query = results.get("query");
-        Assert.assertNotNull(query);
+        assertNotNull(query);
 
         JSONObject dataType = results.getJSONObject("dataType");
-        Assert.assertNotNull(dataType);
+        assertNotNull(dataType);
         String typeName = dataType.getString("typeName");
-        Assert.assertNotNull(typeName);
+        assertNotNull(typeName);
 
         JSONArray rows = results.getJSONArray("rows");
        
-        Assert.assertNotNull(rows);
-        Assert.assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may
not have any results
+        assertNotNull(rows);
+        assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may not
have any results
         List<String> returnedList = new ArrayList<String> ();
         for (int i = 0; i < rows.length(); i++) {
             JSONObject row = rows.getJSONObject(i);
@@ -546,23 +578,23 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest
{
     public void  testSearchByDSLQueries(String dslQuery, Integer expectedNumRows) throws
Exception {
         System.out.println("Executing dslQuery = " + dslQuery);
         String jsonResults = discoveryService.searchByDSL(dslQuery);
-        Assert.assertNotNull(jsonResults);
+        assertNotNull(jsonResults);
 
         JSONObject results = new JSONObject(jsonResults);
-        Assert.assertEquals(results.length(), 3);
+        assertEquals(results.length(), 3);
         System.out.println("results = " + results);
 
         Object query = results.get("query");
-        Assert.assertNotNull(query);
+        assertNotNull(query);
 
         JSONObject dataType = results.getJSONObject("dataType");
-        Assert.assertNotNull(dataType);
+        assertNotNull(dataType);
         String typeName = dataType.getString("typeName");
-        Assert.assertNotNull(typeName);
+        assertNotNull(typeName);
 
         JSONArray rows = results.getJSONArray("rows");
-        Assert.assertNotNull(rows);
-        Assert.assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may
not have any results
+        assertNotNull(rows);
+        assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may not
have any results
         System.out.println("query [" + dslQuery + "] returned [" + rows.length() + "] rows");
     }
 
@@ -570,23 +602,23 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest
{
     public void  testSearchByDSLQueriesWithLimit(String dslQuery, Integer expectedNumRows)
throws Exception {
         System.out.println("Executing dslQuery = " + dslQuery);
         String jsonResults = discoveryService.searchByDSL(dslQuery);
-        Assert.assertNotNull(jsonResults);
+        assertNotNull(jsonResults);
 
         JSONObject results = new JSONObject(jsonResults);
-        Assert.assertEquals(results.length(), 3);
+        assertEquals(results.length(), 3);
         System.out.println("results = " + results);
 
         Object query = results.get("query");
-        Assert.assertNotNull(query);
+        assertNotNull(query);
 
         JSONObject dataType = results.getJSONObject("dataType");
-        Assert.assertNotNull(dataType);
+        assertNotNull(dataType);
         String typeName = dataType.getString("typeName");
-        Assert.assertNotNull(typeName);
+        assertNotNull(typeName);
 
         JSONArray rows = results.getJSONArray("rows");
-        Assert.assertNotNull(rows);
-        Assert.assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may
not have any results
+        assertNotNull(rows);
+        assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may not
have any results
         System.out.println("query [" + dslQuery + "] returned [" + rows.length() + "] rows");
     }
     
@@ -609,7 +641,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest
{
 
         String dslQuery = "from D where a = 1";
         String jsonResults = discoveryService.searchByDSL(dslQuery);
-        Assert.assertNotNull(jsonResults);
+        assertNotNull(jsonResults);
 
         JSONObject results = new JSONObject(jsonResults);
         System.out.println("results = " + results);

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bfd5f5ca/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java
b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java
index 330d1cb..e748579 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java
@@ -38,7 +38,7 @@ public class AttributeInfo {
     public final String reverseAttributeName;
     private IDataType dataType;
 
-    AttributeInfo(TypeSystem t, AttributeDefinition def, Map<String, IDataType> tempTypes)
throws AtlasException {
+    public AttributeInfo(TypeSystem t, AttributeDefinition def, Map<String, IDataType>
tempTypes) throws AtlasException {
         this.name = def.name;
         this.dataType =
                 (tempTypes != null && tempTypes.containsKey(def.dataTypeName)) ?
tempTypes.get(def.dataTypeName) :

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bfd5f5ca/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
b/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
index 2ea90a3..ef8448d 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
@@ -21,8 +21,10 @@ package org.apache.atlas.typesystem.types.utils;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 
+import org.apache.atlas.AtlasException;
 import org.apache.atlas.typesystem.TypesDef;
 import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.AttributeInfo;
 import org.apache.atlas.typesystem.types.ClassType;
 import org.apache.atlas.typesystem.types.EnumTypeDefinition;
 import org.apache.atlas.typesystem.types.EnumValue;
@@ -32,6 +34,7 @@ import org.apache.atlas.typesystem.types.Multiplicity;
 import org.apache.atlas.typesystem.types.StructTypeDefinition;
 import org.apache.atlas.typesystem.types.TraitType;
 
+import org.apache.atlas.typesystem.types.TypeSystem;
 import scala.collection.JavaConversions;
 
 /**
@@ -100,4 +103,15 @@ public class TypesUtil {
         return new TypesDef(JavaConversions.asScalaBuffer(enums), JavaConversions.asScalaBuffer(structs),
                 JavaConversions.asScalaBuffer(traits), JavaConversions.asScalaBuffer(classes));
     }
+
+    private static final TypeSystem ts = TypeSystem.getInstance();
+
+    public static AttributeInfo newAttributeInfo(String attribute, IDataType type) {
+        try {
+            return new AttributeInfo(ts, new AttributeDefinition(attribute, type.getName(),
Multiplicity.REQUIRED,
+                    false, null), null);
+        } catch (AtlasException e) {
+            throw new RuntimeException(e);
+        }
+    }
 }


Mime
View raw message