atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mad...@apache.org
Subject incubator-atlas git commit: ATLAS-1981: Cache escaped type-query string to avoid repeated computation
Date Fri, 21 Jul 2017 19:26:15 GMT
Repository: incubator-atlas
Updated Branches:
  refs/heads/0.8-incubating 47d11cc38 -> f800575ef


ATLAS-1981: Cache escaped type-query string to avoid repeated computation

(cherry picked from commit e0d2cdc27c32742ebecd24db4cca62dc04dcdf4b)


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/f800575e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/f800575e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/f800575e

Branch: refs/heads/0.8-incubating
Commit: f800575eff23f7fd05f3f79b59dd6567b4a33f1c
Parents: 47d11cc
Author: Madhan Neethiraj <madhan@apache.org>
Authored: Fri Jul 21 09:28:44 2017 -0700
Committer: Madhan Neethiraj <madhan@apache.org>
Committed: Fri Jul 21 12:25:39 2017 -0700

----------------------------------------------------------------------
 .../org/apache/atlas/repository/Constants.java  |  4 +-
 .../org/apache/atlas/model/TypeCategory.java    |  2 +-
 .../atlas/type/AtlasClassificationType.java     | 30 ++++++++---
 .../org/apache/atlas/type/AtlasEntityType.java  | 28 +++++++---
 .../org/apache/atlas/type/AtlasStructType.java  | 40 +++++++++++++++
 .../java/org/apache/atlas/type/AtlasType.java   |  3 ++
 .../apache/atlas/type/AtlasTypeRegistry.java    |  4 ++
 .../ClassificationSearchProcessor.java          | 17 +++---
 .../atlas/discovery/EntityDiscoveryService.java | 32 ++++++------
 .../atlas/discovery/EntitySearchProcessor.java  | 18 ++++---
 .../discovery/FullTextSearchProcessor.java      | 23 ++++-----
 .../apache/atlas/discovery/SearchProcessor.java | 54 ++++----------------
 .../graph/v1/AtlasTypeDefGraphStoreV1.java      |  3 ++
 .../typestore/GraphBackedTypeStore.java         |  8 +++
 .../atlas/typesystem/types/DataTypes.java       |  3 +-
 15 files changed, 162 insertions(+), 107 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/common/src/main/java/org/apache/atlas/repository/Constants.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java
index 7156d56..62d74f8 100644
--- a/common/src/main/java/org/apache/atlas/repository/Constants.java
+++ b/common/src/main/java/org/apache/atlas/repository/Constants.java
@@ -92,8 +92,8 @@ public final class Constants {
     public static final String QUALIFIED_NAME = "Referenceable.qualifiedName";
     public static final String TYPE_NAME_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "typeName";
     public static final String INDEX_SEARCH_MAX_RESULT_SET_SIZE = "atlas.graph.index.search.max-result-set-size";
-    public static final String INDEX_SEARCH_MAX_TYPES_COUNT = "atlas.graph.index.search.max-types-count";
-    public static final String INDEX_SEARCH_MAX_TAGS_COUNT = "atlas.graph.index.search.max-tags-count";
+    public static final String INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH = "atlas.graph.index.search.types.max-query-str-length";
+    public static final String INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH  = "atlas.graph.index.search.tags.max-query-str-length";
 
     private Constants() {
     }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/intg/src/main/java/org/apache/atlas/model/TypeCategory.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/TypeCategory.java b/intg/src/main/java/org/apache/atlas/model/TypeCategory.java
index e47a8a7..f06f64f 100644
--- a/intg/src/main/java/org/apache/atlas/model/TypeCategory.java
+++ b/intg/src/main/java/org/apache/atlas/model/TypeCategory.java
@@ -18,5 +18,5 @@
 package org.apache.atlas.model;
 
 public enum TypeCategory {
-    PRIMITIVE, OBJECT_ID_TYPE, ENUM, STRUCT, CLASSIFICATION, ENTITY, ARRAY, MAP
+    PRIMITIVE, OBJECT_ID_TYPE, ENUM, STRUCT, CLASSIFICATION, ENTITY, ARRAY, MAP, RELATIONSHIP
 }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
index c1c3add..56c3ed3 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
@@ -45,10 +45,11 @@ public class AtlasClassificationType extends AtlasStructType {
 
     private final AtlasClassificationDef classificationDef;
 
-    private List<AtlasClassificationType> superTypes         = Collections.emptyList();
-    private Set<String>                   allSuperTypes      = Collections.emptySet();
-    private Set<String>                   allSubTypes        = Collections.emptySet();
-    private Set<String>                   typeAndAllSubTypes = Collections.emptySet();
+    private List<AtlasClassificationType> superTypes               = Collections.emptyList();
+    private Set<String>                   allSuperTypes            = Collections.emptySet();
+    private Set<String>                   allSubTypes              = Collections.emptySet();
+    private Set<String>                   typeAndAllSubTypes       = Collections.emptySet();
+    private String                        typeAndAllSubTypesQryStr = "";
 
     public AtlasClassificationType(AtlasClassificationDef classificationDef) {
         super(classificationDef);
@@ -108,6 +109,13 @@ public class AtlasClassificationType extends AtlasStructType {
         }
     }
 
+    @Override
+    public void resolveReferencesPhase3(AtlasTypeRegistry typeRegistry) throws AtlasBaseException
{
+        allSubTypes              = Collections.unmodifiableSet(allSubTypes);
+        typeAndAllSubTypes       = Collections.unmodifiableSet(typeAndAllSubTypes);
+        typeAndAllSubTypesQryStr = ""; // will be computed on next access
+    }
+
     private void addSubType(AtlasClassificationType subType) {
         allSubTypes.add(subType.getTypeName());
         typeAndAllSubTypes.add(subType.getTypeName());
@@ -119,11 +127,17 @@ public class AtlasClassificationType extends AtlasStructType {
 
     public Set<String> getAllSuperTypes() { return allSuperTypes; }
 
-    public Set<String> getAllSubTypes() {
-        return Collections.unmodifiableSet(allSubTypes);
-    }
+    public Set<String> getAllSubTypes() { return allSubTypes; }
 
-        public Set<String> getTypeAndAllSubTypes() { return Collections.unmodifiableSet(typeAndAllSubTypes);
}
+    public Set<String> getTypeAndAllSubTypes() { return typeAndAllSubTypes; }
+
+    public String getTypeAndAllSubTypesQryStr() {
+        if (StringUtils.isEmpty(typeAndAllSubTypesQryStr)) {
+            typeAndAllSubTypesQryStr = AtlasAttribute.escapeIndexQueryValue(typeAndAllSubTypes);
+        }
+
+        return typeAndAllSubTypesQryStr;
+    }
 
     public boolean isSuperTypeOf(AtlasClassificationType classificationType) {
         return classificationType != null && allSubTypes.contains(classificationType.getTypeName());

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
index 6516d48..2ad8f1c 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
@@ -47,10 +47,11 @@ public class AtlasEntityType extends AtlasStructType {
 
     private final AtlasEntityDef entityDef;
 
-    private List<AtlasEntityType> superTypes         = Collections.emptyList();
-    private Set<String>           allSuperTypes      = Collections.emptySet();
-    private Set<String>           allSubTypes        = Collections.emptySet();
-    private Set<String>           typeAndAllSubTypes = Collections.emptySet();
+    private List<AtlasEntityType> superTypes               = Collections.emptyList();
+    private Set<String>           allSuperTypes            = Collections.emptySet();
+    private Set<String>           allSubTypes              = Collections.emptySet();
+    private Set<String>           typeAndAllSubTypes       = Collections.emptySet();
+    private String                typeAndAllSubTypesQryStr = "";
 
     public AtlasEntityType(AtlasEntityDef entityDef) {
         super(entityDef);
@@ -108,6 +109,13 @@ public class AtlasEntityType extends AtlasStructType {
         }
     }
 
+    @Override
+    public void resolveReferencesPhase3(AtlasTypeRegistry typeRegistry) throws AtlasBaseException
{
+        allSubTypes              = Collections.unmodifiableSet(allSubTypes);
+        typeAndAllSubTypes       = Collections.unmodifiableSet(typeAndAllSubTypes);
+        typeAndAllSubTypesQryStr = ""; // will be computed on next access
+    }
+
     public Set<String> getSuperTypes() {
         return entityDef.getSuperTypes();
     }
@@ -116,9 +124,9 @@ public class AtlasEntityType extends AtlasStructType {
         return allSuperTypes;
     }
 
-    public Set<String> getAllSubTypes() { return Collections.unmodifiableSet(allSubTypes);
}
+    public Set<String> getAllSubTypes() { return allSubTypes; }
 
-    public Set<String> getTypeAndAllSubTypes() { return Collections.unmodifiableSet(typeAndAllSubTypes);
}
+    public Set<String> getTypeAndAllSubTypes() { return typeAndAllSubTypes; }
 
     public boolean isSuperTypeOf(AtlasEntityType entityType) {
         return entityType != null && allSubTypes.contains(entityType.getTypeName());
@@ -136,6 +144,14 @@ public class AtlasEntityType extends AtlasStructType {
         return StringUtils.isNotEmpty(entityTypeName) && allSuperTypes.contains(entityTypeName);
     }
 
+    public String getTypeAndAllSubTypesQryStr() {
+        if (StringUtils.isEmpty(typeAndAllSubTypesQryStr)) {
+            typeAndAllSubTypesQryStr = AtlasAttribute.escapeIndexQueryValue(typeAndAllSubTypes);
+        }
+
+        return typeAndAllSubTypesQryStr;
+    }
+
     @Override
     public AtlasEntity createDefaultValue() {
         AtlasEntity ret = new AtlasEntity(entityDef.getName());

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
index 0eeaf9c..e73b503 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
@@ -39,6 +39,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * class that implements behaviour of a struct-type.
@@ -683,6 +684,39 @@ public class AtlasStructType extends AtlasType {
             return key;
         }
 
+        public static String escapeIndexQueryValue(Set<String> values) {
+            StringBuilder sb = new StringBuilder();
+
+            sb.append(BRACE_OPEN_CHAR);
+            for (String value : values) {
+                sb.append(escapeIndexQueryValue(value)).append(SPACE_CHAR);
+            }
+            sb.append(BRACE_CLOSE_CHAR);
+
+            return sb.toString();
+        }
+
+        public static String escapeIndexQueryValue(String value) {
+            String ret = value;
+
+            if (StringUtils.containsAny(value, IDX_QRY_OFFENDING_CHARS)) {
+                boolean isQuoteAtStart = value.charAt(0) == DOUBLE_QUOTE_CHAR;
+                boolean isQuoteAtEnd   = value.charAt(value.length() - 1) == DOUBLE_QUOTE_CHAR;
+
+                if (!isQuoteAtStart) {
+                    if (!isQuoteAtEnd) {
+                        ret = DOUBLE_QUOTE_CHAR + value + DOUBLE_QUOTE_CHAR;
+                    } else {
+                        ret = DOUBLE_QUOTE_CHAR + value;
+                    }
+                } else if (!isQuoteAtEnd) {
+                    ret = value + DOUBLE_QUOTE_CHAR;
+                }
+            }
+
+            return ret;
+        }
+
         private static String getQualifiedAttributeName(AtlasStructDef structDef, String
attrName) {
             final String typeName = structDef.getName();
             return attrName.contains(".") ? attrName : String.format("%s.%s", typeName, attrName);
@@ -695,5 +729,11 @@ public class AtlasStructType extends AtlasType {
                 new String[] { "$",  "_d" },
                 new String[] { "%", "_p"  },
         };
+
+        private static final char[] IDX_QRY_OFFENDING_CHARS = { '@', '/', ' ' };
+        private static final char   BRACE_OPEN_CHAR         = '(';
+        private static final char   BRACE_CLOSE_CHAR        = ')';
+        private static final char   DOUBLE_QUOTE_CHAR       = '"';
+        private static final char   SPACE_CHAR              = ' ';
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/intg/src/main/java/org/apache/atlas/type/AtlasType.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasType.java b/intg/src/main/java/org/apache/atlas/type/AtlasType.java
index 79de9d5..747c9a6 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasType.java
@@ -57,6 +57,9 @@ public abstract class AtlasType {
     public void resolveReferencesPhase2(AtlasTypeRegistry typeRegistry) throws AtlasBaseException
{
     }
 
+    public void resolveReferencesPhase3(AtlasTypeRegistry typeRegistry) throws AtlasBaseException
{
+    }
+
     public String getTypeName() { return typeName; }
 
     public TypeCategory getTypeCategory() { return typeCategory; }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
index 1b3526b..dbd1038 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
@@ -343,6 +343,10 @@ public class AtlasTypeRegistry {
             for (AtlasType type : registryData.allTypes.getAllTypes()) {
                 type.resolveReferencesPhase2(this);
             }
+
+            for (AtlasType type : registryData.allTypes.getAllTypes()) {
+                type.resolveReferencesPhase3(this);
+            }
         }
 
         public void clear() {

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
index 745f9d7..dcdcbbb 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
@@ -42,23 +42,24 @@ public class ClassificationSearchProcessor extends SearchProcessor {
     public ClassificationSearchProcessor(SearchContext context) {
         super(context);
 
-        AtlasClassificationType classificationType = context.getClassificationType();
-        FilterCriteria          filterCriteria     = context.getSearchParameters().getTagFilters();
-        Set<String>             typeAndSubTypes    = classificationType.getTypeAndAllSubTypes();
-        Set<String>             solrAttributes     = new HashSet<>();
-        Set<String>             gremlinAttributes  = new HashSet<>();
-        Set<String>             allAttributes      = new HashSet<>();
+        final AtlasClassificationType classificationType    = context.getClassificationType();
+        final FilterCriteria          filterCriteria        = context.getSearchParameters().getTagFilters();
+        final Set<String>             typeAndSubTypes       = classificationType.getTypeAndAllSubTypes();
+        final String                  typeAndSubTypesQryStr = classificationType.getTypeAndAllSubTypesQryStr();
+        final Set<String>             solrAttributes        = new HashSet<>();
+        final Set<String>             gremlinAttributes     = new HashSet<>();
+        final Set<String>             allAttributes         = new HashSet<>();
 
 
         processSearchAttributes(classificationType, filterCriteria, solrAttributes, gremlinAttributes,
allAttributes);
 
         // for classification search, if any attribute can't be handled by Solr - switch
to all Gremlin
-        boolean useSolrSearch = typeAndSubTypes.size() <= MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY
&& CollectionUtils.isEmpty(gremlinAttributes) && canApplySolrFilter(classificationType,
filterCriteria, false);
+        boolean useSolrSearch = typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS
&& CollectionUtils.isEmpty(gremlinAttributes) && canApplySolrFilter(classificationType,
filterCriteria, false);
 
         if (useSolrSearch) {
             StringBuilder solrQuery = new StringBuilder();
 
-            constructTypeTestQuery(solrQuery, typeAndSubTypes);
+            constructTypeTestQuery(solrQuery, typeAndSubTypesQryStr);
             constructFilterQuery(solrQuery, classificationType, filterCriteria, solrAttributes);
 
             String solrQueryString = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")");

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
index dbe11a6..b183c72 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
@@ -87,8 +87,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
     private final GraphBackedSearchIndexer        indexer;
     private final SearchTracker                   searchTracker;
     private final int                             maxResultSetSize;
-    private final int                             maxTypesCountInIdxQuery;
-    private final int                             maxTagsCountInIdxQuery;
+    private final int                             maxTypesLengthInIdxQuery;
+    private final int                             maxTagsLengthInIdxQuery;
 
     @Inject
     EntityDiscoveryService(MetadataRepository metadataRepository, AtlasTypeRegistry typeRegistry,
@@ -101,8 +101,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
         this.gremlinQueryProvider     = AtlasGremlinQueryProvider.INSTANCE;
         this.typeRegistry             = typeRegistry;
         this.maxResultSetSize         = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE,
150);
-        this.maxTypesCountInIdxQuery  = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_TYPES_COUNT,
10);
-        this.maxTagsCountInIdxQuery   = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_TAGS_COUNT,
10);
+        this.maxTypesLengthInIdxQuery = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH,
512);
+        this.maxTagsLengthInIdxQuery  = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH,
512);
     }
 
     @Override
@@ -490,8 +490,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
     }
 
     private String getQueryForFullTextSearch(String userKeyedString, String typeName, String
classification) {
-        String typeFilter              = getTypeFilter(typeRegistry, typeName, maxTypesCountInIdxQuery);
-        String classificationFilter = getClassificationFilter(typeRegistry, classification,
maxTagsCountInIdxQuery);
+        String typeFilter           = getTypeFilter(typeRegistry, typeName, maxTypesLengthInIdxQuery);
+        String classificationFilter = getClassificationFilter(typeRegistry, classification,
maxTagsLengthInIdxQuery);
 
         StringBuilder queryText = new StringBuilder();
 
@@ -619,23 +619,23 @@ public class EntityDiscoveryService implements AtlasDiscoveryService
{
         return excludeDeletedEntities && GraphHelper.getStatus(vertex) == Status.DELETED;
     }
 
-    private static String getClassificationFilter(AtlasTypeRegistry typeRegistry, String
classificationName, int maxTypesCountInIdxQuery) {
-        AtlasClassificationType classification  = typeRegistry.getClassificationTypeByName(classificationName);
-        Set<String>             typeAndSubTypes = classification != null ? classification.getTypeAndAllSubTypes()
: null;
+    private static String getClassificationFilter(AtlasTypeRegistry typeRegistry, String
classificationName, int maxTypesLengthInIdxQuery) {
+        AtlasClassificationType type                  = typeRegistry.getClassificationTypeByName(classificationName);
+        String                  typeAndSubTypesQryStr = type != null ? type.getTypeAndAllSubTypesQryStr()
: null;
 
-        if(CollectionUtils.isNotEmpty(typeAndSubTypes) && typeAndSubTypes.size()
<= maxTypesCountInIdxQuery) {
-            return String.format("(%s)", StringUtils.join(typeAndSubTypes, " "));
+        if(StringUtils.isNotEmpty(typeAndSubTypesQryStr) && typeAndSubTypesQryStr.length()
<= maxTypesLengthInIdxQuery) {
+            return typeAndSubTypesQryStr;
         }
 
         return "";
     }
 
-    private static String getTypeFilter(AtlasTypeRegistry typeRegistry, String typeName,
int maxTypesCountInIdxQuery) {
-        AtlasEntityType type            = typeRegistry.getEntityTypeByName(typeName);
-        Set<String>     typeAndSubTypes = type != null ? type.getTypeAndAllSubTypes()
: null;
+    private static String getTypeFilter(AtlasTypeRegistry typeRegistry, String typeName,
int maxTypesLengthInIdxQuery) {
+        AtlasEntityType type                  = typeRegistry.getEntityTypeByName(typeName);
+        String          typeAndSubTypesQryStr = type != null ? type.getTypeAndAllSubTypesQryStr()
: null;
 
-        if(CollectionUtils.isNotEmpty(typeAndSubTypes) && typeAndSubTypes.size()
<= maxTypesCountInIdxQuery) {
-            return String.format("(%s)", StringUtils.join(typeAndSubTypes, " "));
+        if(StringUtils.isNotEmpty(typeAndSubTypesQryStr) && typeAndSubTypesQryStr.length()
<= maxTypesLengthInIdxQuery) {
+            return typeAndSubTypesQryStr;
         }
 
         return "";

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index efcfb7f..385951b 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -40,12 +40,13 @@ public class EntitySearchProcessor extends SearchProcessor {
     public EntitySearchProcessor(SearchContext context) {
         super(context);
 
-        final AtlasEntityType entityType        = context.getEntityType();
-        final FilterCriteria  filterCriteria    = context.getSearchParameters().getEntityFilters();
-        final Set<String>     typeAndSubTypes   = entityType.getTypeAndAllSubTypes();
-        final Set<String>     solrAttributes    = new HashSet<>();
-        final Set<String>     gremlinAttributes = new HashSet<>();
-        final Set<String>     allAttributes     = new HashSet<>();
+        final AtlasEntityType entityType            = context.getEntityType();
+        final FilterCriteria  filterCriteria        = context.getSearchParameters().getEntityFilters();
+        final Set<String>     typeAndSubTypes       = entityType.getTypeAndAllSubTypes();
+        final String          typeAndSubTypesQryStr = entityType.getTypeAndAllSubTypesQryStr();
+        final Set<String>     solrAttributes        = new HashSet<>();
+        final Set<String>     gremlinAttributes     = new HashSet<>();
+        final Set<String>     allAttributes         = new HashSet<>();
 
         final AtlasClassificationType classificationType   = context.getClassificationType();
         final boolean                 filterClassification = classificationType != null &&
!context.needClassificationProcessor();
@@ -53,13 +54,13 @@ public class EntitySearchProcessor extends SearchProcessor {
 
         processSearchAttributes(entityType, filterCriteria, solrAttributes, gremlinAttributes,
allAttributes);
 
-        final boolean typeSearchBySolr = !filterClassification && typeAndSubTypes.size()
<= MAX_ENTITY_TYPES_IN_INDEX_QUERY;
+        final boolean typeSearchBySolr = !filterClassification && typeAndSubTypesQryStr.length()
<= MAX_QUERY_STR_LENGTH_TYPES;
         final boolean attrSearchBySolr = !filterClassification && CollectionUtils.isNotEmpty(solrAttributes)
&& canApplySolrFilter(entityType, filterCriteria, false);
 
         StringBuilder solrQuery = new StringBuilder();
 
         if (typeSearchBySolr) {
-            constructTypeTestQuery(solrQuery, typeAndSubTypes);
+            constructTypeTestQuery(solrQuery, typeAndSubTypesQryStr);
         }
 
         if (attrSearchBySolr) {
@@ -169,6 +170,7 @@ public class EntitySearchProcessor extends SearchProcessor {
 
                         guidQuery.addConditionsFrom(graphQuery);
 
+                        entityVertices.clear();
                         getVertices(guidQuery.vertices().iterator(), entityVertices);
                     }
                 } else {

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
index 0d1c39b..d556bf1 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
@@ -30,7 +30,6 @@ import org.slf4j.LoggerFactory;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Set;
 
 
 public class FullTextSearchProcessor extends SearchProcessor {
@@ -50,28 +49,26 @@ public class FullTextSearchProcessor extends SearchProcessor {
         // if search includes entity-type criteria, adding a filter here can help avoid unnecessary
         // processing (and rejection) by subsequent EntitySearchProcessor
         if (context.getEntityType() != null) {
-            Set<String> typeAndSubTypeNames = context.getEntityType().getTypeAndAllSubTypes();
+            String typeAndSubTypeNamesQryStr = context.getEntityType().getTypeAndAllSubTypesQryStr();
 
-            if (typeAndSubTypeNames.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY) {
-                queryString.append(AND_STR);
-                appendIndexQueryValue(typeAndSubTypeNames, queryString);
+            if (typeAndSubTypeNamesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES) {
+                queryString.append(AND_STR).append(typeAndSubTypeNamesQryStr);
             } else {
-                LOG.warn("'{}' has too many subtypes ({}) to include in index-query; might
cause poor performance",
-                         context.getEntityType().getTypeName(), typeAndSubTypeNames.size());
+                LOG.warn("'{}' has too many subtypes (query-string-length={}) to include
in index-query; might cause poor performance",
+                         context.getEntityType().getTypeName(), typeAndSubTypeNamesQryStr.length());
             }
         }
 
         // if search includes classification criteria, adding a filter here can help avoid
unnecessary
         // processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor
         if (context.getClassificationType() != null) {
-            Set<String> typeAndSubTypeNames = context.getClassificationType().getTypeAndAllSubTypes();
+            String typeAndSubTypeNamesStr = context.getClassificationType().getTypeAndAllSubTypesQryStr();
 
-            if (typeAndSubTypeNames.size() <= MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY)
{
-                queryString.append(AND_STR);
-                appendIndexQueryValue(typeAndSubTypeNames, queryString);
+            if (typeAndSubTypeNamesStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) {
+                queryString.append(AND_STR).append(typeAndSubTypeNamesStr);
             } else {
-                LOG.warn("'{}' has too many subtypes ({}) to include in index-query; might
cause poor performance",
-                        context.getClassificationType().getTypeName(), typeAndSubTypeNames.size());
+                LOG.warn("'{}' has too many subtypes (query-string-length={}) to include
in index-query; might cause poor performance",
+                        context.getClassificationType().getTypeName(), typeAndSubTypeNamesStr.length());
             }
         }
 

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index 2862165..7e09672 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -39,21 +39,19 @@ import java.util.regex.Pattern;
 public abstract class SearchProcessor {
     private static final Logger LOG = LoggerFactory.getLogger(SearchProcessor.class);
 
-    public static final Pattern STRAY_AND_PATTERN                       = Pattern.compile("(AND\\s+)+\\)");
-    public static final Pattern STRAY_OR_PATTERN                        = Pattern.compile("(OR\\s+)+\\)");
-    public static final Pattern STRAY_ELIPSIS_PATTERN                   = Pattern.compile("(\\(\\s*)\\)");
-    public static final int     MAX_RESULT_SIZE                         = getApplicationProperty(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE,
150);
-    public static final int     MAX_ENTITY_TYPES_IN_INDEX_QUERY         = getApplicationProperty(Constants.INDEX_SEARCH_MAX_TYPES_COUNT,
10);
-    public static final int     MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY = getApplicationProperty(Constants.INDEX_SEARCH_MAX_TAGS_COUNT,
10);
+    public static final Pattern STRAY_AND_PATTERN          = Pattern.compile("(AND\\s+)+\\)");
+    public static final Pattern STRAY_OR_PATTERN           = Pattern.compile("(OR\\s+)+\\)");
+    public static final Pattern STRAY_ELIPSIS_PATTERN      = Pattern.compile("(\\(\\s*)\\)");
+    public static final int     MAX_RESULT_SIZE            = getApplicationProperty(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE,
150);
+    public static final int     MAX_QUERY_STR_LENGTH_TYPES = getApplicationProperty(Constants.INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH,
512);
+    public static final int     MAX_QUERY_STR_LENGTH_TAGS  = getApplicationProperty(Constants.INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH,
512);
     public static final String  AND_STR         = " AND ";
     public static final String  EMPTY_STRING    = "";
     public static final String  SPACE_STRING    = " ";
     public static final String  BRACE_OPEN_STR  = "(";
     public static final String  BRACE_CLOSE_STR = ")";
-    public static final char    DOUBLE_QUOTE    = '"';
 
     private static final Map<SearchParameters.Operator, String> OPERATOR_MAP = new
HashMap<>();
-    private static final char[] OFFENDING_CHARS = { '@', '/', ' ' }; // This can grow as
we discover corner cases
 
     static
     {
@@ -181,14 +179,13 @@ public abstract class SearchProcessor {
         return ret;
     }
 
-    protected void constructTypeTestQuery(StringBuilder solrQuery, Set<String> typeAndAllSubTypes)
{
-        if (CollectionUtils.isNotEmpty(typeAndAllSubTypes)) {
+    protected void constructTypeTestQuery(StringBuilder solrQuery, String typeAndAllSubTypesQryStr)
{
+        if (StringUtils.isNotEmpty(typeAndAllSubTypesQryStr)) {
             if (solrQuery.length() > 0) {
                 solrQuery.append(AND_STR);
             }
 
-            solrQuery.append("v.\"").append(Constants.TYPE_NAME_PROPERTY_KEY).append("\":");
-            appendIndexQueryValue(typeAndAllSubTypes, solrQuery);
+            solrQuery.append("v.\"").append(Constants.TYPE_NAME_PROPERTY_KEY).append("\":").append(typeAndAllSubTypesQryStr);
         }
     }
 
@@ -255,7 +252,7 @@ public abstract class SearchProcessor {
             if (OPERATOR_MAP.get(op) != null) {
                 String qualifiedName = type.getQualifiedAttributeName(attrName);
 
-                ret = String.format(OPERATOR_MAP.get(op), qualifiedName, escapeIndexQueryValue(attrVal));
+                ret = String.format(OPERATOR_MAP.get(op), qualifiedName, AtlasStructType.AtlasAttribute.escapeIndexQueryValue(attrVal));
             }
         } catch (AtlasBaseException ex) {
             LOG.warn(ex.getMessage());
@@ -389,16 +386,6 @@ public abstract class SearchProcessor {
         return ret;
     }
 
-    protected String appendIndexQueryValue(Set<String> values, StringBuilder sb) {
-        sb.append(BRACE_OPEN_STR);
-        for (String value : values) {
-            sb.append(escapeIndexQueryValue(value)).append(SPACE_STRING);
-        }
-        sb.append(BRACE_CLOSE_STR);
-
-        return sb.toString();
-    }
-
     private static int getApplicationProperty(String propertyName, int defaultValue) {
         try {
             return ApplicationProperties.get().getInt(propertyName, defaultValue);
@@ -408,25 +395,4 @@ public abstract class SearchProcessor {
 
         return defaultValue;
     }
-
-    private String escapeIndexQueryValue(String value) {
-        String ret = value;
-
-        if (StringUtils.containsAny(value, OFFENDING_CHARS)) {
-            boolean isQuoteAtStart = value.charAt(0) == DOUBLE_QUOTE;
-            boolean isQuoteAtEnd   = value.charAt(value.length() - 1) == DOUBLE_QUOTE;
-
-            if (!isQuoteAtStart) {
-                if (!isQuoteAtEnd) {
-                    ret = DOUBLE_QUOTE + value + DOUBLE_QUOTE;
-                } else {
-                    ret = DOUBLE_QUOTE + value;
-                }
-            } else if (!isQuoteAtEnd) {
-                ret = value + DOUBLE_QUOTE;
-            }
-        }
-
-        return ret;
-    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java
index f0c8380..fbd6314 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java
@@ -385,6 +385,9 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
 
             case ENUM:
                 return TypeCategory.ENUM;
+
+            case RELATIONSHIP:
+                return TypeCategory.RELATIONSHIP;
         }
 
         return null;

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
index 7a064b6..1525d7b 100644
--- a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
+++ b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
@@ -107,6 +107,10 @@ public class GraphBackedTypeStore implements ITypeStore {
                         type.superTypes, visitor);
                 break;
 
+            case RELATIONSHIP:
+                // ignore
+                break;
+
             default:    //Ignore primitive/collection types as they are covered under references
                 break;
             }
@@ -273,6 +277,10 @@ public class GraphBackedTypeStore implements ITypeStore {
                 traits.add(new HierarchicalTypeDefinition(TraitType.class, typeName, typeDescription,
superTypes, attributes));
                 break;
 
+            case RELATIONSHIP:
+                // v1 typesystem is not notified on new relation type
+                break;
+
             default:
                 throw new IllegalArgumentException("Unhandled type category " + typeCategory);
             }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java
index 21d5f1a..f9f4abe 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java
@@ -86,7 +86,8 @@ public class DataTypes {
         MAP,
         STRUCT,
         TRAIT,
-        CLASS
+        CLASS,
+        RELATIONSHIP
     }
 
     public static abstract class PrimitiveType<T> extends AbstractDataType<T>
{



Mime
View raw message