atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sar...@apache.org
Subject incubator-atlas git commit: ATLAS-1807 : Enhance DSL query to support like operator for wildcard search
Date Wed, 17 May 2017 00:33:27 GMT
Repository: incubator-atlas
Updated Branches:
  refs/heads/master cf64a03e0 -> f2255da11


ATLAS-1807 : Enhance DSL query to support like operator for wildcard search


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/f2255da1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/f2255da1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/f2255da1

Branch: refs/heads/master
Commit: f2255da116af44da4c1c86c585fe7428d218691a
Parents: cf64a03
Author: Sarath Subramanian <ssubramanian@hortonworks.com>
Authored: Tue May 16 17:33:02 2017 -0700
Committer: Sarath Subramanian <ssubramanian@hortonworks.com>
Committed: Tue May 16 17:33:02 2017 -0700

----------------------------------------------------------------------
 .../gremlin/Gremlin2ExpressionFactory.java      | 32 ++++++++++++++++++++
 .../gremlin/Gremlin3ExpressionFactory.java      | 32 ++++++++++++++++++++
 .../atlas/gremlin/GremlinExpressionFactory.java |  4 +++
 .../org/apache/atlas/query/GremlinQuery.scala   |  4 +++
 .../org/apache/atlas/query/QueryParser.scala    |  3 +-
 .../GraphBackedDiscoveryServiceTest.java        | 16 ++++++++++
 .../EntityDiscoveryJerseyResourceIT.java        | 18 ++++++++++-
 7 files changed, 107 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f2255da1/repository/src/main/java/org/apache/atlas/gremlin/Gremlin2ExpressionFactory.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/gremlin/Gremlin2ExpressionFactory.java
b/repository/src/main/java/org/apache/atlas/gremlin/Gremlin2ExpressionFactory.java
index 807dd05..5f8bb80 100644
--- a/repository/src/main/java/org/apache/atlas/gremlin/Gremlin2ExpressionFactory.java
+++ b/repository/src/main/java/org/apache/atlas/gremlin/Gremlin2ExpressionFactory.java
@@ -40,6 +40,7 @@ import org.apache.atlas.groovy.LogicalExpression.LogicalOperator;
 import org.apache.atlas.groovy.RangeExpression;
 import org.apache.atlas.groovy.TernaryOperatorExpression;
 import org.apache.atlas.groovy.TraversalStepType;
+import org.apache.atlas.query.Expressions;
 import org.apache.atlas.query.GraphPersistenceStrategies;
 import org.apache.atlas.query.TypeUtils.FieldInfo;
 import org.apache.atlas.typesystem.types.IDataType;
@@ -147,6 +148,37 @@ public class Gremlin2ExpressionFactory extends GremlinExpressionFactory
{
         return new FunctionCallExpression(TraversalStepType.FILTER, parent, HAS_METHOD, propertyNameExpr,
op, requiredValue);
     }
 
+    @Override
+    public GroovyExpression generateLikeExpressionUsingFilter(GroovyExpression parent, String
propertyName, GroovyExpression propertyValue) throws AtlasException {
+        GroovyExpression itExpr      = getItVariable();
+        GroovyExpression nameExpr    = new FieldExpression(itExpr, propertyName);
+        GroovyExpression matchesExpr = new FunctionCallExpression(nameExpr, MATCHES, escapePropertyValue(propertyValue));
+        GroovyExpression closureExpr = new ClosureExpression(matchesExpr);
+        GroovyExpression filterExpr  = new FunctionCallExpression(parent, FILTER_METHOD,
closureExpr);
+
+        return filterExpr;
+    }
+
+    private GroovyExpression escapePropertyValue(GroovyExpression propertyValue) {
+        GroovyExpression ret = propertyValue;
+
+        if (propertyValue instanceof LiteralExpression) {
+            LiteralExpression exp = (LiteralExpression) propertyValue;
+
+            if (exp != null && exp.getValue() instanceof String) {
+                String stringValue = (String) exp.getValue();
+
+                // replace '*' with ".*", replace '?' with '.'
+                stringValue = stringValue.replaceAll("\\*", ".*")
+                                         .replaceAll("\\?", ".");
+
+                ret = new LiteralExpression(stringValue);
+            }
+        }
+
+        return ret;
+    }
+
     private GroovyExpression gremlin2CompOp(String op) throws AtlasException {
 
         GroovyExpression tExpr = new IdentifierExpression("T");

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f2255da1/repository/src/main/java/org/apache/atlas/gremlin/Gremlin3ExpressionFactory.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/gremlin/Gremlin3ExpressionFactory.java
b/repository/src/main/java/org/apache/atlas/gremlin/Gremlin3ExpressionFactory.java
index 9f68c9a..aaec6fe 100644
--- a/repository/src/main/java/org/apache/atlas/gremlin/Gremlin3ExpressionFactory.java
+++ b/repository/src/main/java/org/apache/atlas/gremlin/Gremlin3ExpressionFactory.java
@@ -29,6 +29,7 @@ import org.apache.atlas.groovy.ClosureExpression;
 import org.apache.atlas.groovy.ComparisonExpression;
 import org.apache.atlas.groovy.ComparisonExpression.ComparisonOperator;
 import org.apache.atlas.groovy.ComparisonOperatorExpression;
+import org.apache.atlas.groovy.FieldExpression;
 import org.apache.atlas.groovy.FunctionCallExpression;
 import org.apache.atlas.groovy.GroovyExpression;
 import org.apache.atlas.groovy.IdentifierExpression;
@@ -244,6 +245,37 @@ public class Gremlin3ExpressionFactory extends GremlinExpressionFactory
{
     }
 
     @Override
+    public GroovyExpression generateLikeExpressionUsingFilter(GroovyExpression parent, String
propertyName, GroovyExpression propertyValue) throws AtlasException {
+        GroovyExpression itExpr      = getItVariable();
+        GroovyExpression nameExpr    = new FieldExpression(itExpr, propertyName);
+        GroovyExpression matchesExpr = new FunctionCallExpression(nameExpr, MATCHES, escapePropertyValue(propertyValue));
+        GroovyExpression closureExpr = new ClosureExpression(matchesExpr);
+        GroovyExpression filterExpr  = new FunctionCallExpression(parent, FILTER_METHOD,
closureExpr);
+
+        return filterExpr;
+    }
+
+    private GroovyExpression escapePropertyValue(GroovyExpression propertyValue) {
+        GroovyExpression ret = propertyValue;
+
+        if (propertyValue instanceof LiteralExpression) {
+            LiteralExpression exp = (LiteralExpression) propertyValue;
+
+            if (exp != null && exp.getValue() instanceof String) {
+                String stringValue = (String) exp.getValue();
+
+                // replace '*' with ".*", replace '?' with '.'
+                stringValue = stringValue.replaceAll("\\*", ".*")
+                        .replaceAll("\\?", ".");
+
+                ret = new LiteralExpression(stringValue);
+            }
+        }
+
+        return ret;
+    }
+
+    @Override
     protected GroovyExpression initialExpression(GroovyExpression varExpr, GraphPersistenceStrategies
s) {
 
         // this bit of groovy magic converts the set of vertices in varName into

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f2255da1/repository/src/main/java/org/apache/atlas/gremlin/GremlinExpressionFactory.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/gremlin/GremlinExpressionFactory.java
b/repository/src/main/java/org/apache/atlas/gremlin/GremlinExpressionFactory.java
index ff5a58c..d603150 100644
--- a/repository/src/main/java/org/apache/atlas/gremlin/GremlinExpressionFactory.java
+++ b/repository/src/main/java/org/apache/atlas/gremlin/GremlinExpressionFactory.java
@@ -76,6 +76,7 @@ public abstract class GremlinExpressionFactory {
     protected static final String SELECT_METHOD = "select";
     protected static final String ORDER_METHOD = "order";
     protected static final String FILL_METHOD = "fill";
+    protected static final String MATCHES = "matches";
 
     public static final GremlinExpressionFactory INSTANCE = AtlasGraphProvider.getGraphInstance()
             .getSupportedGremlinVersion() == GremlinVersion.THREE ? new Gremlin3ExpressionFactory()
@@ -182,6 +183,9 @@ public abstract class GremlinExpressionFactory {
     public abstract GroovyExpression generateHasExpression(GraphPersistenceStrategies s,
GroovyExpression parent,
             String propertyName, String symbol, GroovyExpression requiredValue, FieldInfo
fInfo) throws AtlasException;
 
+    public abstract GroovyExpression generateLikeExpressionUsingFilter(GroovyExpression parent,
String propertyName,
+                                                                       GroovyExpression propertyValue)
throws AtlasException;
+
     /**
      * Generates a range expression
      *

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f2255da1/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala b/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala
index 3a310a7..37015d8 100644
--- a/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala
+++ b/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala
@@ -437,6 +437,10 @@ class GremlinTranslator(expr: Expression,
                 genQuery(null, l, inClosure);
             }
 
+            if (symb == "like") {
+              return GremlinExpressionFactory.INSTANCE.generateLikeExpressionUsingFilter(childExpr,
qualifiedPropertyName, persistentExprValue);
+            }
+
            return GremlinExpressionFactory.INSTANCE.generateHasExpression(gPersistenceBehavior,
childExpr, qualifiedPropertyName, c.symbol, persistentExprValue, fInfo);
         }
         case fil@FilterExpression(child, condExpr) => {

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f2255da1/repository/src/main/scala/org/apache/atlas/query/QueryParser.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/atlas/query/QueryParser.scala b/repository/src/main/scala/org/apache/atlas/query/QueryParser.scala
index 8d454e9..7b7cd98 100755
--- a/repository/src/main/scala/org/apache/atlas/query/QueryParser.scala
+++ b/repository/src/main/scala/org/apache/atlas/query/QueryParser.scala
@@ -75,6 +75,7 @@ trait QueryKeywords {
     protected val SUM = Keyword("sum")
     protected val BY = Keyword("by")
     protected val ORDER = Keyword("order")
+    protected val LIKE = Keyword("like")
 }
 
 trait ExpressionUtils {
@@ -312,7 +313,7 @@ object QueryParser extends StandardTokenParsers with QueryKeywords with
Expressi
     def exprRight = (AND | OR) ~ compE ^^ { case op ~ c => (op, c)}
 
     def compE =
-        arithE ~ (LT | LTE | EQ | NEQ | GT | GTE) ~ arithE ^^ { case l ~ op ~ r => l.compareOp(op)(r)}
|
+        arithE ~ (LT | LTE | EQ | NEQ | GT | GTE | LIKE) ~ arithE ^^ { case l ~ op ~ r =>
l.compareOp(op)(r)} |
             arithE ~ (ISA | IS) ~ ident ^^ { case l ~ i ~ t => l.isTrait(t)} |
             arithE ~ HAS ~ ident ^^ { case l ~ i ~ f => l.hasField(f)} |
             arithE | countClause | maxClause | minClause | sumClause

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f2255da1/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
index 18573fc..3ea8f34 100755
--- a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
@@ -256,6 +256,22 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest
{
         assertEquals(entityState, Id.EntityState.ACTIVE.name());
     }
 
+    @DataProvider(name = "dslLikeQueriesProvider")
+    private Object[][] createDslLikeQueries() {
+        return new Object[][]{
+                {"hive_table where name like \"sa?es*\"", 3},
+                {"hive_db where name like \"R*\"", 1},
+                {"hive_db where hive_db.name like \"R???rt?*\" or hive_db.name like \"S?l?s\"
or hive_db.name like\"Log*\"", 3},
+                {"hive_db where hive_db.name like \"R???rt?*\" and hive_db.name like \"S?l?s\"
and hive_db.name like\"Log*\"", 0},
+                {"hive_table where name like 'sales*', db where name like 'Sa?es'", 1},
+        };
+    }
+
+    @Test(dataProvider = "dslLikeQueriesProvider")
+    public void testDslSearchUsingLikeOperator(String dslQuery, Integer expectedNumRows)
throws Exception {
+        runQuery(dslQuery, expectedNumRows, 50, 0);
+    }
+
     @Test(expectedExceptions = Throwable.class)
     public void testSearchByDSLBadQuery() throws Exception {
         String dslQuery = "from blah";

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f2255da1/webapp/src/test/java/org/apache/atlas/web/resources/EntityDiscoveryJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/EntityDiscoveryJerseyResourceIT.java
b/webapp/src/test/java/org/apache/atlas/web/resources/EntityDiscoveryJerseyResourceIT.java
index db3a7c3..a51f371 100755
--- a/webapp/src/test/java/org/apache/atlas/web/resources/EntityDiscoveryJerseyResourceIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/EntityDiscoveryJerseyResourceIT.java
@@ -54,7 +54,7 @@ public class EntityDiscoveryJerseyResourceIT extends BaseResourceIT {
     @BeforeClass
     public void setUp() throws Exception {
         super.setUp();
-        dbName = "db" + randomString();
+        dbName = "database" + randomString();
         createTypes();
         createInstance(createHiveDBInstanceBuiltIn(dbName));
     }
@@ -145,6 +145,22 @@ public class EntityDiscoveryJerseyResourceIT extends BaseResourceIT {
     }
 
     @Test
+    public void testLikeSearchUsingDSL() throws Exception {
+        String dslQuery = DATABASE_TYPE_BUILTIN + " where " + QUALIFIED_NAME + " like \"da?a*\"";
+
+        AtlasSearchResult searchResult = atlasClientV2.dslSearch(dslQuery);
+        assertNotNull(searchResult);
+
+        List<AtlasEntityHeader> entities = searchResult.getEntities();
+        assertNotNull(entities);
+        assertEquals(entities.size(), 1);
+
+        AtlasEntityHeader dbEntity = entities.get(0);
+        assertEquals(dbEntity.getTypeName(), DATABASE_TYPE_BUILTIN);
+        assertEquals(dbEntity.getDisplayText(), dbName);
+    }
+
+    @Test
     public void testSearchFullTextOnDSLFailure() throws Exception {
         String query = "*";
         AtlasSearchResult searchResult = atlasClientV2.fullTextSearch(query);


Mime
View raw message