atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apoorvn...@apache.org
Subject atlas git commit: ATLAS-2174: Query length validations and path normalization
Date Mon, 02 Oct 2017 16:41:37 GMT
Repository: atlas
Updated Branches:
  refs/heads/branch-0.8 28941bfe7 -> b30444e5d


ATLAS-2174: Query length validations and path normalization

Signed-off-by: apoorvnaik <apoorvnaik@apache.org>
(cherry picked from commit 657e0f1272d1361dacee61f3bf328f8a3185cfab)


Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/b30444e5
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/b30444e5
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/b30444e5

Branch: refs/heads/branch-0.8
Commit: b30444e5d328c3e6b4f0ba79cd2dcba8a3664188
Parents: 28941bf
Author: nixonrodrigues <nixon@apache.org>
Authored: Mon Oct 2 09:03:50 2017 -0700
Committer: apoorvnaik <apoorvnaik@apache.org>
Committed: Mon Oct 2 09:41:21 2017 -0700

----------------------------------------------------------------------
 .../main/java/org/apache/atlas/AtlasClient.java |  6 ++--
 .../java/org/apache/atlas/AtlasClientTest.java  |  1 +
 .../java/org/apache/atlas/AtlasBaseClient.java  | 21 ++++++++-----
 .../org/apache/atlas/AtlasServiceException.java |  2 +-
 .../org/apache/atlas/repository/Constants.java  |  3 ++
 .../java/org/apache/atlas/AtlasErrorCode.java   |  1 +
 .../notification/NotificationHookConsumer.java  |  8 ++---
 .../atlas/web/resources/EntityResource.java     |  2 +-
 .../apache/atlas/web/rest/DiscoveryREST.java    | 32 +++++++++++++++-----
 9 files changed, 51 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/atlas/blob/b30444e5/client/client-v1/src/main/java/org/apache/atlas/AtlasClient.java
----------------------------------------------------------------------
diff --git a/client/client-v1/src/main/java/org/apache/atlas/AtlasClient.java b/client/client-v1/src/main/java/org/apache/atlas/AtlasClient.java
index 6d0b83d..8bbc89b 100644
--- a/client/client-v1/src/main/java/org/apache/atlas/AtlasClient.java
+++ b/client/client-v1/src/main/java/org/apache/atlas/AtlasClient.java
@@ -356,7 +356,7 @@ public class AtlasClient extends AtlasBaseClient {
         JSONObject response = callAPIWithRetries(api, null, new ResourceCreator() {
             @Override
             public WebResource createResource() {
-                WebResource resource = getResource(api.getPath());
+                WebResource resource = getResource(api.getNormalizedPath());
                 resource = resource.queryParam(TYPE, category.name());
                 return resource;
             }
@@ -924,12 +924,12 @@ public class AtlasClient extends AtlasBaseClient {
 
     @VisibleForTesting
     public WebResource getResource(API api, String... params) {
-        return getResource(api.getPath(), params);
+        return getResource(api.getNormalizedPath(), params);
     }
 
     @VisibleForTesting
     public WebResource getResource(API_V1 apiV1, String... params) {
-        return getResource(apiV1.getPath(), params);
+        return getResource(apiV1.getNormalizedPath(), params);
     }
 
     @VisibleForTesting

http://git-wip-us.apache.org/repos/asf/atlas/blob/b30444e5/client/client-v1/src/test/java/org/apache/atlas/AtlasClientTest.java
----------------------------------------------------------------------
diff --git a/client/client-v1/src/test/java/org/apache/atlas/AtlasClientTest.java b/client/client-v1/src/test/java/org/apache/atlas/AtlasClientTest.java
index 820fe27..732cea0 100644
--- a/client/client-v1/src/test/java/org/apache/atlas/AtlasClientTest.java
+++ b/client/client-v1/src/test/java/org/apache/atlas/AtlasClientTest.java
@@ -104,6 +104,7 @@ public class AtlasClientTest {
 
     private WebResource.Builder setupBuilder(AtlasClient.API_V1 api, WebResource webResource)
{
         when(webResource.path(api.getPath())).thenReturn(service);
+        when(webResource.path(api.getNormalizedPath())).thenReturn(service);
         return getBuilder(service);
     }
 

http://git-wip-us.apache.org/repos/asf/atlas/blob/b30444e5/client/common/src/main/java/org/apache/atlas/AtlasBaseClient.java
----------------------------------------------------------------------
diff --git a/client/common/src/main/java/org/apache/atlas/AtlasBaseClient.java b/client/common/src/main/java/org/apache/atlas/AtlasBaseClient.java
index 5e1d101..1616029 100644
--- a/client/common/src/main/java/org/apache/atlas/AtlasBaseClient.java
+++ b/client/common/src/main/java/org/apache/atlas/AtlasBaseClient.java
@@ -46,6 +46,7 @@ import javax.ws.rs.core.Response;
 import javax.ws.rs.core.UriBuilder;
 import java.io.IOException;
 import java.net.ConnectException;
+import java.nio.file.Paths;
 import java.util.List;
 import java.util.Map;
 
@@ -151,7 +152,7 @@ public abstract class AtlasBaseClient {
     }
 
     public boolean isServerReady() throws AtlasServiceException {
-        WebResource resource   = getResource(API_VERSION.getPath());
+        WebResource resource   = getResource(API_VERSION.getNormalizedPath());
         try {
             callAPIWithResource(API_VERSION, resource, null, JSONObject.class);
             return true;
@@ -175,7 +176,7 @@ public abstract class AtlasBaseClient {
      */
     public String getAdminStatus() throws AtlasServiceException {
         String      result    = AtlasBaseClient.UNKNOWN_STATUS;
-        WebResource resource  = getResource(service, API_STATUS.getPath());
+        WebResource resource  = getResource(service, API_STATUS.getNormalizedPath());
         JSONObject  response  = callAPIWithResource(API_STATUS, resource, null, JSONObject.class);
         try {
             result = response.getString("Status");
@@ -315,7 +316,7 @@ public abstract class AtlasBaseClient {
         int i = 0;
         do {
             if (LOG.isDebugEnabled()) {
-                LOG.debug("Calling API [ {} : {} ] {}", api.getMethod(), api.getPath(), requestObject
!= null ? "<== " + requestObject : "");
+                LOG.debug("Calling API [ {} : {} ] {}", api.getMethod(), api.getNormalizedPath(),
requestObject != null ? "<== " + requestObject : "");
             }
 
             WebResource.Builder requestBuilder = resource.getRequestBuilder();
@@ -375,7 +376,7 @@ public abstract class AtlasBaseClient {
     }
 
     protected WebResource getResource(API api, MultivaluedMap<String, String> queryParams,
String... pathParams) {
-        WebResource resource = service.path(api.getPath());
+        WebResource resource = service.path(api.getNormalizedPath());
         resource = appendPathParams(resource, pathParams);
         resource = appendQueryParams(queryParams, resource);
         return resource;
@@ -447,7 +448,7 @@ public abstract class AtlasBaseClient {
                 if (i == (getNumberOfRetries() - 1)) {
                     throw che;
                 }
-                LOG.warn("Handled exception in calling api {}", api.getPath(), che);
+                LOG.warn("Handled exception in calling api {}", api.getNormalizedPath(),
che);
                 LOG.warn("Exception's cause: {}", che.getCause().getClass());
                 handleClientHandlerException(che);
             }
@@ -515,13 +516,13 @@ public abstract class AtlasBaseClient {
 
     // Modify URL to include the path params
     private WebResource getResource(WebResource service, API api, String... pathParams) {
-        WebResource resource = service.path(api.getPath());
+        WebResource resource = service.path(api.getNormalizedPath());
         resource = appendPathParams(resource, pathParams);
         return resource;
     }
 
     private WebResource getResource(API api, String queryParamKey, List<String> queryParamValues)
{
-        WebResource resource = service.path(api.getPath());
+        WebResource resource = service.path(api.getNormalizedPath());
         for (String queryParamValue : queryParamValues) {
             if (StringUtils.isNotBlank(queryParamKey) && StringUtils.isNotBlank(queryParamValue))
{
                 resource = resource.queryParam(queryParamKey, queryParamValue);
@@ -541,7 +542,7 @@ public abstract class AtlasBaseClient {
 
     // Modify URL to include the query params
     private WebResource getResource(WebResource service, API api, MultivaluedMap<String,
String> queryParams) {
-        WebResource resource = service.path(api.getPath());
+        WebResource resource = service.path(api.getNormalizedPath());
         resource = appendQueryParams(queryParams, resource);
         return resource;
     }
@@ -578,6 +579,10 @@ public abstract class AtlasBaseClient {
             return path;
         }
 
+        public String getNormalizedPath() {
+            return Paths.get(path).normalize().toString();
+        }
+
         public Response.Status getExpectedStatus() {
             return status;
         }

http://git-wip-us.apache.org/repos/asf/atlas/blob/b30444e5/client/common/src/main/java/org/apache/atlas/AtlasServiceException.java
----------------------------------------------------------------------
diff --git a/client/common/src/main/java/org/apache/atlas/AtlasServiceException.java b/client/common/src/main/java/org/apache/atlas/AtlasServiceException.java
index 83f4f8d..33d0a21 100755
--- a/client/common/src/main/java/org/apache/atlas/AtlasServiceException.java
+++ b/client/common/src/main/java/org/apache/atlas/AtlasServiceException.java
@@ -28,7 +28,7 @@ public class AtlasServiceException extends Exception {
     private ClientResponse.Status status;
 
     public AtlasServiceException(AtlasBaseClient.API api, Exception e) {
-        super("Metadata service API " + api.getMethod() + " : " + api.getPath() + " failed",
e);
+        super("Metadata service API " + api.getMethod() + " : " + api.getNormalizedPath()
+ " failed", e);
     }
 
     public AtlasServiceException(AtlasBaseClient.API api, WebApplicationException e) throws
JSONException {

http://git-wip-us.apache.org/repos/asf/atlas/blob/b30444e5/common/src/main/java/org/apache/atlas/repository/Constants.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java
index 1817929..4e179fb 100644
--- a/common/src/main/java/org/apache/atlas/repository/Constants.java
+++ b/common/src/main/java/org/apache/atlas/repository/Constants.java
@@ -96,6 +96,9 @@ public final class Constants {
     public static final String INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH = "atlas.graph.index.search.types.max-query-str-length";
     public static final String INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH  = "atlas.graph.index.search.tags.max-query-str-length";
 
+    public static final String MAX_FULLTEXT_QUERY_STR_LENGTH  = "atlas.graph.fulltext-max-query-str-length";
+    public static final String MAX_DSL_QUERY_STR_LENGTH  = "atlas.graph.dsl-max-query-str-length";
+
     private Constants() {
     }
 

http://git-wip-us.apache.org/repos/asf/atlas/blob/b30444e5/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
index 18392d4..2d340b1 100644
--- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
+++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
@@ -76,6 +76,7 @@ public enum AtlasErrorCode {
     INVALID_IMPORT_ATTRIBUTE_TYPE_CHANGED(400, "ATLAS-400-00-050", "Attribute {0}.{1} is
of type {2}. Import has this attribute type as {3}"),
     SAVED_SEARCH_CHANGE_USER(400, "ATLAS-400-00-056", "saved-search {0} can not be moved
from user {1} to {2}"),
     INVALID_QUERY_PARAM_LENGTH(400, "ATLAS-400-00-057" , "Length of query param {0} exceeds
the limit"),
+    INVALID_QUERY_LENGTH(400, "ATLAS-400-00-057" , "Invalid query length, update {0} to change
the limit" ),
 
     // All Not found enums go here
     UNKNOWN_CLASSIFICATION(400, "ATLAS-400-00-046", "{0}: Unknown/invalid classification"),

http://git-wip-us.apache.org/repos/asf/atlas/blob/b30444e5/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java
b/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java
index 36be03e..4e0a55d 100644
--- a/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java
+++ b/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java
@@ -353,7 +353,7 @@ public class NotificationHookConsumer implements Service, ActiveStateChangeHandl
 
                                 if (numRetries == 0) { // audit only on the first attempt
                                     AtlasBaseClient.API api = AtlasClient.API_V1.CREATE_ENTITY;
-                                    audit(messageUser, api.getMethod(), api.getPath());
+                                    audit(messageUser, api.getMethod(), api.getNormalizedPath());
                                 }
 
                                 entities = instanceConverter.toAtlasEntities(createRequest.getEntities());
@@ -367,7 +367,7 @@ public class NotificationHookConsumer implements Service, ActiveStateChangeHandl
                                 if (numRetries == 0) { // audit only on the first attempt
                                     AtlasBaseClient.API api = UPDATE_ENTITY_BY_ATTRIBUTE;
                                     audit(messageUser, api.getMethod(),
-                                          String.format(api.getPath(), partialUpdateRequest.getTypeName()));
+                                          String.format(api.getNormalizedPath(), partialUpdateRequest.getTypeName()));
                                 }
 
                                 Referenceable referenceable = partialUpdateRequest.getEntity();
@@ -392,7 +392,7 @@ public class NotificationHookConsumer implements Service, ActiveStateChangeHandl
                                 if (numRetries == 0) { // audit only on the first attempt
                                     AtlasBaseClient.API api = DELETE_ENTITY_BY_ATTRIBUTE;
                                     audit(messageUser, api.getMethod(),
-                                          String.format(api.getPath(), deleteRequest.getTypeName()));
+                                          String.format(api.getNormalizedPath(), deleteRequest.getTypeName()));
                                 }
 
                                 try {
@@ -411,7 +411,7 @@ public class NotificationHookConsumer implements Service, ActiveStateChangeHandl
 
                                 if (numRetries == 0) { // audit only on the first attempt
                                     AtlasBaseClient.API api = UPDATE_ENTITY;
-                                    audit(messageUser, api.getMethod(), api.getPath());
+                                    audit(messageUser, api.getMethod(), api.getNormalizedPath());
                                 }
 
                                 entities = instanceConverter.toAtlasEntities(updateRequest.getEntities());

http://git-wip-us.apache.org/repos/asf/atlas/blob/b30444e5/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java
index 67c9b27..8b56507 100755
--- a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java
@@ -215,7 +215,7 @@ public class EntityResource {
             UriBuilder ub = uriInfo.getAbsolutePathBuilder();
             locationURI = CollectionUtils.isEmpty(guids) ? null : ub.path(guids.get(0)).build();
         } else {
-            String uriPath = AtlasClient.API_V1.GET_ENTITY.getPath();
+            String uriPath = AtlasClient.API_V1.GET_ENTITY.getNormalizedPath();
             locationURI = guids.isEmpty() ? null : UriBuilder
                 .fromPath(AtlasConstants.DEFAULT_ATLAS_REST_ADDRESS)
                 .path(uriPath).path(guids.get(0)).build();

http://git-wip-us.apache.org/repos/asf/atlas/blob/b30444e5/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
index 1780c67..ee68d63 100644
--- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
+++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
@@ -17,7 +17,6 @@
  */
 package org.apache.atlas.web.rest;
 
-import org.apache.atlas.AtlasConfiguration;
 import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.SortOrder;
 import org.apache.atlas.discovery.AtlasDiscoveryService;
@@ -25,9 +24,11 @@ import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.discovery.AtlasSearchResult;
 import org.apache.atlas.model.discovery.SearchParameters;
 import org.apache.atlas.model.profile.AtlasUserSavedSearch;
+import org.apache.atlas.repository.Constants;
 import org.apache.atlas.utils.AtlasPerfTracer;
 import org.apache.atlas.web.util.Servlets;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.configuration.Configuration;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.springframework.stereotype.Service;
@@ -58,13 +59,17 @@ public class DiscoveryREST {
     private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.DiscoveryREST");
 
     @Context
-    private HttpServletRequest httpServletRequest;
+    private       HttpServletRequest httpServletRequest;
+    private final int                maxFullTextQueryLength;
+    private final int                maxDslQueryLength;
 
     private final AtlasDiscoveryService atlasDiscoveryService;
 
     @Inject
-    public DiscoveryREST(AtlasDiscoveryService discoveryService) {
-        this.atlasDiscoveryService = discoveryService;
+    public DiscoveryREST(AtlasDiscoveryService atlasDiscoveryService, Configuration configuration)
{
+        this.atlasDiscoveryService = atlasDiscoveryService;
+        maxFullTextQueryLength = configuration.getInt(Constants.MAX_FULLTEXT_QUERY_STR_LENGTH,
4096);
+        maxDslQueryLength = configuration.getInt(Constants.MAX_DSL_QUERY_STR_LENGTH, 4096);
     }
 
     /**
@@ -90,10 +95,13 @@ public class DiscoveryREST {
                                             @QueryParam("classification") String classification,
                                             @QueryParam("limit")          int    limit,
                                             @QueryParam("offset")         int    offset)
throws AtlasBaseException {
-        Servlets.validateQueryParamLength("query", query);
         Servlets.validateQueryParamLength("typeName", typeName);
         Servlets.validateQueryParamLength("classification", classification);
 
+        if (StringUtils.isNotEmpty(query) && query.length() > maxDslQueryLength)
{
+            throw new AtlasBaseException(AtlasErrorCode.INVALID_QUERY_LENGTH, Constants.MAX_DSL_QUERY_STR_LENGTH);
+        }
+
         AtlasPerfTracer perf = null;
 
         try {
@@ -132,7 +140,10 @@ public class DiscoveryREST {
                                                  @QueryParam("excludeDeletedEntities") boolean
excludeDeletedEntities,
                                                  @QueryParam("limit")                  int
    limit,
                                                  @QueryParam("offset")                 int
    offset) throws AtlasBaseException {
-        Servlets.validateQueryParamLength("query", query);
+        // Validate FullText query for max allowed length
+        if(StringUtils.isNotEmpty(query) && query.length() > maxFullTextQueryLength){
+            throw new AtlasBaseException(AtlasErrorCode.INVALID_QUERY_LENGTH, Constants.MAX_FULLTEXT_QUERY_STR_LENGTH
);
+        }
 
         AtlasPerfTracer perf = null;
 
@@ -172,9 +183,11 @@ public class DiscoveryREST {
                                               @QueryParam("excludeDeletedEntities") boolean
excludeDeletedEntities,
                                               @QueryParam("limit")                  int 
   limit,
                                               @QueryParam("offset")                 int 
   offset) throws AtlasBaseException {
-        Servlets.validateQueryParamLength("query", query);
         Servlets.validateQueryParamLength("typeName", typeName);
         Servlets.validateQueryParamLength("classification", classification);
+        if (StringUtils.isNotEmpty(query) && query.length() > maxFullTextQueryLength)
{
+            throw new AtlasBaseException(AtlasErrorCode.INVALID_QUERY_LENGTH, Constants.MAX_FULLTEXT_QUERY_STR_LENGTH);
+        }
 
         AtlasPerfTracer perf = null;
 
@@ -556,7 +569,10 @@ public class DiscoveryREST {
         if (parameters != null) {
             Servlets.validateQueryParamLength("typeName", parameters.getTypeName());
             Servlets.validateQueryParamLength("classification", parameters.getClassification());
-            Servlets.validateQueryParamLength("query", parameters.getQuery());
+            if (StringUtils.isNotEmpty(parameters.getQuery()) && parameters.getQuery().length()
> maxFullTextQueryLength) {
+                throw new AtlasBaseException(AtlasErrorCode.INVALID_QUERY_LENGTH, Constants.MAX_FULLTEXT_QUERY_STR_LENGTH);
+            }
+
         }
     }
 }


Mime
View raw message