Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 5C2AB200C76 for ; Sat, 29 Apr 2017 02:44:07 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 5ABA7160BB8; Sat, 29 Apr 2017 00:44:07 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 532BE160BA3 for ; Sat, 29 Apr 2017 02:44:06 +0200 (CEST) Received: (qmail 55917 invoked by uid 500); 29 Apr 2017 00:44:05 -0000 Mailing-List: contact commits-help@atlas.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@atlas.incubator.apache.org Delivered-To: mailing list commits@atlas.incubator.apache.org Received: (qmail 55908 invoked by uid 99); 29 Apr 2017 00:44:05 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd4-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 29 Apr 2017 00:44:05 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd4-us-west.apache.org (ASF Mail Server at spamd4-us-west.apache.org) with ESMTP id 1AC84C031B for ; Sat, 29 Apr 2017 00:44:05 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd4-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -4.221 X-Spam-Level: X-Spam-Status: No, score=-4.221 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-0.001, SPF_PASS=-0.001, URIBL_BLOCKED=0.001] autolearn=disabled Received: from mx1-lw-eu.apache.org ([10.40.0.8]) by localhost (spamd4-us-west.apache.org [10.40.0.11]) (amavisd-new, port 10024) with ESMTP id NMiFJmxPKCkn for ; Sat, 29 Apr 2017 00:44:02 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-eu.apache.org (ASF Mail Server at mx1-lw-eu.apache.org) with SMTP id 277035F5F9 for ; Sat, 29 Apr 2017 00:44:00 +0000 (UTC) Received: (qmail 55861 invoked by uid 99); 29 Apr 2017 00:44:00 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 29 Apr 2017 00:44:00 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 3D6C2E02B4; Sat, 29 Apr 2017 00:44:00 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sarath@apache.org To: commits@atlas.incubator.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: incubator-atlas git commit: ATLAS-1742: Provide option to exclude deleted entities in basic and fulltext search Date: Sat, 29 Apr 2017 00:44:00 +0000 (UTC) archived-at: Sat, 29 Apr 2017 00:44:07 -0000 Repository: incubator-atlas Updated Branches: refs/heads/0.8-incubating f32436b64 -> cba4e4d54 ATLAS-1742: Provide option to exclude deleted entities in basic and fulltext search (cherry picked from commit ca64ef4e80fe621435b93888abb2feee712a6283) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/cba4e4d5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/cba4e4d5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/cba4e4d5 Branch: refs/heads/0.8-incubating Commit: cba4e4d543cc474271d6cfe7a1815d48055af8d5 Parents: f32436b Author: Sarath Subramanian Authored: Fri Apr 28 17:42:02 2017 -0700 Committer: Sarath Subramanian Committed: Fri Apr 28 17:43:49 2017 -0700 ---------------------------------------------------------------------- .../atlas/discovery/AtlasDiscoveryService.java | 6 ++-- .../atlas/discovery/EntityDiscoveryService.java | 34 ++++++++++++++++---- .../apache/atlas/web/rest/DiscoveryREST.java | 27 +++++++++------- 3 files changed, 46 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/cba4e4d5/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java index 30d62a4..923a198 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java @@ -35,11 +35,12 @@ public interface AtlasDiscoveryService { /** * * @param query search query. + * @param excludeDeletedEntities exclude deleted entities in search result. * @param limit number of resultant rows (for pagination). [ limit > 0 ] and [ limit < maxlimit ]. -1 maps to atlas.search.defaultlimit property. * @param offset offset to the results returned (for pagination). [ offset >= 0 ]. -1 maps to offset 0. * @return AtlasSearchResult */ - AtlasSearchResult searchUsingFullTextQuery(String query, int limit, int offset) throws AtlasBaseException; + AtlasSearchResult searchUsingFullTextQuery(String query, boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException; /** * @@ -48,10 +49,11 @@ public interface AtlasDiscoveryService { * @param classification classification name. * @param attrName attribute name. * @param attrValuePrefix attribute value prefix. + * @param excludeDeletedEntities exclude deleted entities in search result. * @param limit number of resultant rows (for pagination). [ limit > 0 ] and [ limit < maxlimit ]. -1 maps to atlas.search.defaultlimit property. * @param offset offset to the results returned (for pagination). [ offset >= 0 ]. -1 maps to offset 0. * @return AtlasSearchResult */ AtlasSearchResult searchUsingBasicQuery(String query, String type, String classification, String attrName, - String attrValuePrefix, int limit, int offset) throws AtlasBaseException; + String attrValuePrefix, boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/cba4e4d5/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 07190db..f3f6ee2 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -25,6 +25,7 @@ import org.apache.atlas.model.discovery.AtlasSearchResult.AttributeSearchResult; import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.discovery.AtlasSearchResult; +import org.apache.atlas.model.instance.AtlasEntity.Status; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.query.Expressions.AliasExpression; import org.apache.atlas.query.Expressions.Expression; @@ -46,7 +47,6 @@ import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.v1.EntityGraphRetriever; import org.apache.atlas.type.AtlasClassificationType; import org.apache.atlas.type.AtlasEntityType; -import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasStructType.AtlasAttribute; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.util.AtlasGremlinQueryProvider; @@ -145,7 +145,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { } @Override - public AtlasSearchResult searchUsingFullTextQuery(String fullTextQuery, int limit, int offset) throws AtlasBaseException { + public AtlasSearchResult searchUsingFullTextQuery(String fullTextQuery, boolean excludeDeletedEntities, int limit, int offset) + throws AtlasBaseException { AtlasSearchResult ret = new AtlasSearchResult(fullTextQuery, AtlasQueryType.FULL_TEXT); QueryParams params = validateSearchParams(limit, offset); AtlasIndexQuery idxQuery = toAtlasIndexQuery(fullTextQuery); @@ -153,14 +154,16 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { if (LOG.isDebugEnabled()) { LOG.debug("Executing Full text query: {}", fullTextQuery); } - ret.setFullTextResult(getIndexQueryResults(idxQuery, params)); + ret.setFullTextResult(getIndexQueryResults(idxQuery, params, excludeDeletedEntities)); return ret; } @Override public AtlasSearchResult searchUsingBasicQuery(String query, String typeName, String classification, String attrName, - String attrValuePrefix, int limit, int offset) throws AtlasBaseException { + String attrValuePrefix, boolean excludeDeletedEntities, int limit, + int offset) throws AtlasBaseException { + AtlasSearchResult ret = new AtlasSearchResult(AtlasQueryType.BASIC); if (LOG.isDebugEnabled()) { @@ -267,6 +270,10 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { } } + if (skipDeletedEntities(excludeDeletedEntities, vertex)) { + continue; + } + resultIdx++; if (resultIdx <= startIdx) { @@ -314,6 +321,10 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { if (firstElement instanceof AtlasVertex) { for (Object element : queryResult) { if (element instanceof AtlasVertex) { + if (skipDeletedEntities(excludeDeletedEntities, (AtlasVertex) element)) { + continue; + } + ret.addEntity(entityRetriever.toAtlasEntityHeader((AtlasVertex) element)); } else { LOG.warn("searchUsingBasicQuery({}): expected an AtlasVertex; found unexpected entry in result {}", basicQuery, element); @@ -331,13 +342,18 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { return ret; } - private List getIndexQueryResults(AtlasIndexQuery query, QueryParams params) throws AtlasBaseException { + private List getIndexQueryResults(AtlasIndexQuery query, QueryParams params, boolean excludeDeletedEntities) throws AtlasBaseException { List ret = new ArrayList<>(); Iterator iter = query.vertices(); while (iter.hasNext() && ret.size() < params.limit()) { - Result idxQueryResult = iter.next(); - AtlasVertex vertex = idxQueryResult.getVertex(); + Result idxQueryResult = iter.next(); + AtlasVertex vertex = idxQueryResult.getVertex(); + + if (skipDeletedEntities(excludeDeletedEntities, vertex)) { + continue; + } + String guid = vertex != null ? vertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class) : null; if (guid != null) { @@ -422,4 +438,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { return ret; } + + private boolean skipDeletedEntities(boolean excludeDeletedEntities, AtlasVertex vertex) { + return excludeDeletedEntities && GraphHelper.getStatus(vertex) == Status.DELETED; + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/cba4e4d5/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java index 76ad88c..81dbe25 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java @@ -114,6 +114,7 @@ public class DiscoveryREST { @Consumes(Servlets.JSON_MEDIA_TYPE) @Produces(Servlets.JSON_MEDIA_TYPE) public AtlasSearchResult searchUsingFullText(@QueryParam("query") String query, + @QueryParam("excludeDeletedEntities") boolean excludeDeletedEntities, @QueryParam("limit") int limit, @QueryParam("offset") int offset) throws AtlasBaseException { AtlasPerfTracer perf = null; @@ -124,7 +125,7 @@ public class DiscoveryREST { limit + "," + offset + ")"); } - return atlasDiscoveryService.searchUsingFullTextQuery(query, limit, offset); + return atlasDiscoveryService.searchUsingFullTextQuery(query, excludeDeletedEntities, limit, offset); } finally { AtlasPerfTracer.log(perf); } @@ -147,11 +148,12 @@ public class DiscoveryREST { @Path("/basic") @Consumes(Servlets.JSON_MEDIA_TYPE) @Produces(Servlets.JSON_MEDIA_TYPE) - public AtlasSearchResult searchUsingBasic(@QueryParam("query") String query, - @QueryParam("typeName") String typeName, - @QueryParam("classification") String classification, - @QueryParam("limit") int limit, - @QueryParam("offset") int offset) throws AtlasBaseException { + public AtlasSearchResult searchUsingBasic(@QueryParam("query") String query, + @QueryParam("typeName") String typeName, + @QueryParam("classification") String classification, + @QueryParam("excludeDeletedEntities") boolean excludeDeletedEntities, + @QueryParam("limit") int limit, + @QueryParam("offset") int offset) throws AtlasBaseException { AtlasPerfTracer perf = null; try { @@ -160,7 +162,8 @@ public class DiscoveryREST { typeName + "," + classification + "," + limit + "," + offset + ")"); } - return atlasDiscoveryService.searchUsingBasicQuery(query, typeName, classification, null, null, limit, offset); + return atlasDiscoveryService.searchUsingBasicQuery(query, typeName, classification, null, null, + excludeDeletedEntities, limit, offset); } finally { AtlasPerfTracer.log(perf); } @@ -183,11 +186,11 @@ public class DiscoveryREST { @Path("/attribute") @Consumes(Servlets.JSON_MEDIA_TYPE) @Produces(Servlets.JSON_MEDIA_TYPE) - public AtlasSearchResult searchUsingAttribute(@QueryParam("attrName") String attrName, + public AtlasSearchResult searchUsingAttribute(@QueryParam("attrName") String attrName, @QueryParam("attrValuePrefix") String attrValuePrefix, - @QueryParam("typeName") String typeName, - @QueryParam("limit") int limit, - @QueryParam("offset") int offset) throws AtlasBaseException { + @QueryParam("typeName") String typeName, + @QueryParam("limit") int limit, + @QueryParam("offset") int offset) throws AtlasBaseException { AtlasPerfTracer perf = null; try { @@ -201,7 +204,7 @@ public class DiscoveryREST { String.format("attrName : {0}, attrValue: {1} for attribute search.", attrName, attrValuePrefix)); } - return atlasDiscoveryService.searchUsingBasicQuery(null, typeName, null, attrName, attrValuePrefix, limit, offset); + return atlasDiscoveryService.searchUsingBasicQuery(null, typeName, null, attrName, attrValuePrefix, true, limit, offset); } finally { AtlasPerfTracer.log(perf);