atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shweth...@apache.org
Subject incubator-atlas git commit: ATLAS-1081 Atlas jetty server configuration (shwethags)
Date Tue, 15 Nov 2016 08:24:23 GMT
Repository: incubator-atlas
Updated Branches:
  refs/heads/master d95775161 -> 43a1e25bd


ATLAS-1081 Atlas jetty server configuration (shwethags)


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/43a1e25b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/43a1e25b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/43a1e25b

Branch: refs/heads/master
Commit: 43a1e25bdd1d3f7700c905a29c21dcd601591872
Parents: d957751
Author: Shwetha GS <sshivalingamurthy@hortonworks.com>
Authored: Tue Nov 15 13:54:16 2016 +0530
Committer: Shwetha GS <sshivalingamurthy@hortonworks.com>
Committed: Tue Nov 15 13:54:16 2016 +0530

----------------------------------------------------------------------
 .../org/apache/atlas/AtlasConfiguration.java    | 72 ++++++++++++++++++++
 .../java/org/apache/atlas/AtlasProperties.java  | 64 -----------------
 distro/src/bin/atlas_start.py                   |  2 +-
 docs/src/site/twiki/Configuration.twiki         | 12 ++++
 release-log.txt                                 |  1 +
 .../atlas/discovery/DataSetLineageService.java  |  4 +-
 .../resources/MetadataDiscoveryResource.java    |  6 +-
 .../atlas/web/service/EmbeddedServer.java       | 35 +++++-----
 .../atlas/web/service/SecureEmbeddedServer.java |  9 +--
 9 files changed, 112 insertions(+), 93 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/43a1e25b/common/src/main/java/org/apache/atlas/AtlasConfiguration.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/atlas/AtlasConfiguration.java b/common/src/main/java/org/apache/atlas/AtlasConfiguration.java
new file mode 100644
index 0000000..f5a648d
--- /dev/null
+++ b/common/src/main/java/org/apache/atlas/AtlasConfiguration.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas;
+
+import org.apache.commons.configuration.Configuration;
+
+/**
+ * Enum that encapsulated each property name and its default value.
+ */
+public enum AtlasConfiguration {
+    //web server configuration
+    WEBSERVER_MIN_THREADS("atlas.webserver.minthreads", 10),
+    WEBSERVER_MAX_THREADS("atlas.webserver.maxthreads", 100),
+    WEBSERVER_KEEPALIVE_SECONDS("atlas.webserver.keepalivetimesecs", 60),
+    WEBSERVER_QUEUE_SIZE("atlas.webserver.queuesize", 100),
+    WEBSERVER_REQUEST_BUFFER_SIZE("atlas.jetty.request.buffer.size", 16192),
+
+    //search configuration
+    SEARCH_MAX_LIMIT("atlas.search.maxlimit", 10000),
+    SEARCH_DEFAULT_LIMIT("atlas.search.defaultlimit", 100);
+
+    private static final Configuration APPLICATION_PROPERTIES;
+
+    static {
+        try {
+            APPLICATION_PROPERTIES = ApplicationProperties.get();
+        } catch (AtlasException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private final String propertyName;
+    private final Object defaultValue;
+
+    AtlasConfiguration(String propertyName, Object defaultValue) {
+        this.propertyName = propertyName;
+        this.defaultValue = defaultValue;
+    }
+
+    public int getInt() {
+        return APPLICATION_PROPERTIES.getInt(propertyName, Integer.valueOf(defaultValue.toString()).intValue());
+    }
+
+    public long getLong() {
+        return APPLICATION_PROPERTIES.getLong(propertyName, Long.valueOf(defaultValue.toString()).longValue());
+    }
+
+    public String getString() {
+        return APPLICATION_PROPERTIES.getString(propertyName, defaultValue.toString());
+    }
+
+    public Object get() {
+        Object value = APPLICATION_PROPERTIES.getProperty(propertyName);
+        return value == null ? defaultValue : value;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/43a1e25b/common/src/main/java/org/apache/atlas/AtlasProperties.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/atlas/AtlasProperties.java b/common/src/main/java/org/apache/atlas/AtlasProperties.java
deleted file mode 100644
index df1bccb..0000000
--- a/common/src/main/java/org/apache/atlas/AtlasProperties.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas;
-
-import org.apache.commons.configuration.Configuration;
-
-/**
- * Utility for reading properties in atlas-application.properties.
- */
-public final class AtlasProperties {
-    private static final Configuration APPLICATION_PROPERTIES;
-
-    private AtlasProperties() { }
-
-    static {
-        try {
-            APPLICATION_PROPERTIES = ApplicationProperties.get();
-        } catch (AtlasException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    /**
-     * Enum that encapsulated each property name and its default value.
-     */
-    public enum AtlasProperty {
-        SEARCH_MAX_LIMIT("atlas.search.maxlimit", 10000),
-        SEARCH_DEFAULT_LIMIT("atlas.search.defaultlimit", 100);
-
-        private final String propertyName;
-        private final Object defaultValue;
-
-        AtlasProperty(String propertyName, Object defaultValue) {
-            this.propertyName = propertyName;
-            this.defaultValue = defaultValue;
-        }
-    }
-
-    public static <T> T getProperty(AtlasProperty property) {
-        Object value = APPLICATION_PROPERTIES.getProperty(property.propertyName);
-        if (value == null) {
-            return (T) property.defaultValue;
-        } else {
-            return (T) value;
-
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/43a1e25b/distro/src/bin/atlas_start.py
----------------------------------------------------------------------
diff --git a/distro/src/bin/atlas_start.py b/distro/src/bin/atlas_start.py
index 4590548..a6a3455 100755
--- a/distro/src/bin/atlas_start.py
+++ b/distro/src/bin/atlas_start.py
@@ -25,7 +25,7 @@ ATLAS_LOG_OPTS="-Datlas.log.dir=%s -Datlas.log.file=%s.log"
 ATLAS_COMMAND_OPTS="-Datlas.home=%s"
 ATLAS_CONFIG_OPTS="-Datlas.conf=%s"
 DEFAULT_JVM_HEAP_OPTS="-Xmx1024m -XX:MaxPermSize=512m"
-DEFAULT_JVM_OPTS="-Dlog4j.configuration=atlas-log4j.xml -Djava.net.preferIPv4Stack=true"
+DEFAULT_JVM_OPTS="-Dlog4j.configuration=atlas-log4j.xml -Djava.net.preferIPv4Stack=true -server"
 
 def main():
 

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/43a1e25b/docs/src/site/twiki/Configuration.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/Configuration.twiki b/docs/src/site/twiki/Configuration.twiki
index bbe56d2..7f24f88 100644
--- a/docs/src/site/twiki/Configuration.twiki
+++ b/docs/src/site/twiki/Configuration.twiki
@@ -271,4 +271,16 @@ atlas.graph.storage.lock.retries=10
 # Milliseconds to wait before evicting a cached entry. This should be > atlas.graph.storage.lock.wait-time
x atlas.graph.storage.lock.retries
 # If this is set to a low value (default is 10000), warnings on transactions taking too long
will occur in the Atlas application log.
 atlas.graph.storage.cache.db-cache-time=120000
+
+# Minimum number of threads in the atlas web server
+atlas.webserver.minthreads=10
+
+# Maximum number of threads in the atlas web server
+atlas.webserver.maxthreads=100
+
+# Keepalive time in secs for the thread pool of the atlas web server
+atlas.webserver.keepalivetimesecs=60
+
+# Queue size for the requests(when max threads are busy) for the atlas web server
+atlas.webserver.queuesize=100
 </verbatim>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/43a1e25b/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index da45215..4451222 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements
for al
 ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
 
 ALL CHANGES:
+ATLAS-1081 Atlas jetty server configuration (shwethags)
 ATLAS-1257 Map Entity REST APIs to ATLAS v1 backend (sumasai)
 ATLAS-1279 Remove QueryPlan attribute from Hive Process entity (svimal2106)
 ATLAS-1234 Lineage REST API - v2 (sarath.kum4r@gmail.com via mneethiraj)

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/43a1e25b/repository/src/main/java/org/apache/atlas/discovery/DataSetLineageService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/DataSetLineageService.java
b/repository/src/main/java/org/apache/atlas/discovery/DataSetLineageService.java
index c3fd72b..4d3177c 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/DataSetLineageService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/DataSetLineageService.java
@@ -20,8 +20,8 @@ package org.apache.atlas.discovery;
 
 import org.apache.atlas.ApplicationProperties;
 import org.apache.atlas.AtlasClient;
+import org.apache.atlas.AtlasConfiguration;
 import org.apache.atlas.AtlasException;
-import org.apache.atlas.AtlasProperties;
 import org.apache.atlas.GraphTransaction;
 import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy;
 import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
@@ -178,7 +178,7 @@ public class DataSetLineageService implements LineageService {
         if (propertiesConf.getString(configName) != null) {
             final String schemaQuery =
                 String.format(propertiesConf.getString(configName), guid);
-            int limit = AtlasProperties.getProperty(AtlasProperties.AtlasProperty.SEARCH_MAX_LIMIT);
+            int limit = AtlasConfiguration.SEARCH_MAX_LIMIT.getInt();
             return discoveryService.searchByDSL(schemaQuery, new QueryParams(limit, 0));
         }
         throw new SchemaNotFoundException("Schema is not configured for type " + typeName
+ ". Configure " + configName);

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/43a1e25b/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java
b/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java
index 64344da..ac89869 100755
--- a/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java
@@ -20,7 +20,7 @@ package org.apache.atlas.web.resources;
 
 import com.google.common.base.Preconditions;
 import org.apache.atlas.AtlasClient;
-import org.apache.atlas.AtlasProperties;
+import org.apache.atlas.AtlasConfiguration;
 import org.apache.atlas.classification.InterfaceAudience;
 import org.apache.atlas.discovery.DiscoveryException;
 import org.apache.atlas.discovery.DiscoveryService;
@@ -157,8 +157,8 @@ public class MetadataDiscoveryResource {
     }
 
     private QueryParams validateQueryParams(int limitParam, int offsetParam) {
-        int maxLimit = AtlasProperties.getProperty(AtlasProperties.AtlasProperty.SEARCH_MAX_LIMIT);
-        int defaultLimit = AtlasProperties.getProperty(AtlasProperties.AtlasProperty.SEARCH_DEFAULT_LIMIT);
+        int maxLimit = AtlasConfiguration.SEARCH_MAX_LIMIT.getInt();
+        int defaultLimit = AtlasConfiguration.SEARCH_DEFAULT_LIMIT.getInt();
 
         int limit = defaultLimit;
         boolean limitSet = (limitParam != Integer.valueOf(LIMIT_OFFSET_DEFAULT));

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/43a1e25b/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java b/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java
index 1ee13fb..e615a5b 100755
--- a/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java
+++ b/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java
@@ -18,18 +18,20 @@
 
 package org.apache.atlas.web.service;
 
-import org.apache.atlas.ApplicationProperties;
-import org.apache.commons.configuration.Configuration;
+import org.apache.atlas.AtlasConfiguration;
 import org.eclipse.jetty.server.Connector;
 import org.eclipse.jetty.server.HttpConfiguration;
 import org.eclipse.jetty.server.HttpConnectionFactory;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.util.thread.ExecutorThreadPool;
 import org.eclipse.jetty.webapp.WebAppContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
 
 /**
  * This class embeds a Jetty server and a connector.
@@ -37,11 +39,19 @@ import java.io.IOException;
 public class EmbeddedServer {
     public static final Logger LOG = LoggerFactory.getLogger(EmbeddedServer.class);
 
-    private static final int DEFAULT_BUFFER_SIZE = 16192;
-
-    protected final Server server = new Server();
+    protected final Server server;
 
     public EmbeddedServer(int port, String path) throws IOException {
+        int queueSize = AtlasConfiguration.WEBSERVER_QUEUE_SIZE.getInt();
+        LinkedBlockingQueue<Runnable> queue = new LinkedBlockingQueue<>(queueSize);
+
+        int minThreads = AtlasConfiguration.WEBSERVER_MIN_THREADS.getInt();
+        int maxThreads = AtlasConfiguration.WEBSERVER_MAX_THREADS.getInt();
+        long keepAliveTime = AtlasConfiguration.WEBSERVER_KEEPALIVE_SECONDS.getLong();
+        ExecutorThreadPool pool =
+                new ExecutorThreadPool(minThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS,
queue);
+        server = new Server(pool);
+
         Connector connector = getConnector(port);
         server.addConnector(connector);
 
@@ -66,31 +76,18 @@ public class EmbeddedServer {
     }
 
     protected Connector getConnector(int port) throws IOException {
-
         HttpConfiguration http_config = new HttpConfiguration();
         // this is to enable large header sizes when Kerberos is enabled with AD
-        final int bufferSize = getBufferSize();
+        final int bufferSize = AtlasConfiguration.WEBSERVER_REQUEST_BUFFER_SIZE.getInt();;
         http_config.setResponseHeaderSize(bufferSize);
         http_config.setRequestHeaderSize(bufferSize);
 
         ServerConnector connector = new ServerConnector(server, new HttpConnectionFactory(http_config));
         connector.setPort(port);
         connector.setHost("0.0.0.0");
-        server.addConnector(connector);
         return connector;
     }
 
-    protected Integer getBufferSize() {
-        try {
-            Configuration configuration = ApplicationProperties.get();
-            return configuration.getInt("atlas.jetty.request.buffer.size", DEFAULT_BUFFER_SIZE);
-        } catch (Exception e) {
-            // do nothing
-        }
-
-        return DEFAULT_BUFFER_SIZE;
-    }
-
     public void start() throws Exception {
         server.start();
         server.join();

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/43a1e25b/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java b/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java
index b433e71..a6af5a2 100755
--- a/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java
+++ b/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java
@@ -19,10 +19,12 @@
 package org.apache.atlas.web.service;
 
 import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.AtlasConfiguration;
 import org.apache.atlas.AtlasException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.alias.CredentialProvider;
 import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.eclipse.jetty.http.HttpVersion;
 import org.eclipse.jetty.server.Connector;
 import org.eclipse.jetty.server.HttpConfiguration;
 import org.eclipse.jetty.server.HttpConnectionFactory;
@@ -30,24 +32,23 @@ import org.eclipse.jetty.server.SecureRequestCustomizer;
 import org.eclipse.jetty.server.ServerConnector;
 import org.eclipse.jetty.server.SslConnectionFactory;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
-import org.eclipse.jetty.http.HttpVersion;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.List;
 
+import static org.apache.atlas.security.SecurityProperties.ATLAS_SSL_EXCLUDE_CIPHER_SUITES;
 import static org.apache.atlas.security.SecurityProperties.CERT_STORES_CREDENTIAL_PROVIDER_PATH;
 import static org.apache.atlas.security.SecurityProperties.CLIENT_AUTH_KEY;
 import static org.apache.atlas.security.SecurityProperties.DEFATULT_TRUSTORE_FILE_LOCATION;
+import static org.apache.atlas.security.SecurityProperties.DEFAULT_CIPHER_SUITES;
 import static org.apache.atlas.security.SecurityProperties.DEFAULT_KEYSTORE_FILE_LOCATION;
 import static org.apache.atlas.security.SecurityProperties.KEYSTORE_FILE_KEY;
 import static org.apache.atlas.security.SecurityProperties.KEYSTORE_PASSWORD_KEY;
 import static org.apache.atlas.security.SecurityProperties.SERVER_CERT_PASSWORD_KEY;
 import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_FILE_KEY;
 import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_PASSWORD_KEY;
-import static org.apache.atlas.security.SecurityProperties.ATLAS_SSL_EXCLUDE_CIPHER_SUITES;
-import static org.apache.atlas.security.SecurityProperties.DEFAULT_CIPHER_SUITES;
 
 /**
  * This is a jetty server which requires client auth via certificates.
@@ -81,7 +82,7 @@ public class SecureEmbeddedServer extends EmbeddedServer {
         // HTTP Configuration
         HttpConfiguration http_config = new HttpConfiguration();
         http_config.setSecureScheme("https");
-        final int bufferSize = getBufferSize();
+        final int bufferSize = AtlasConfiguration.WEBSERVER_REQUEST_BUFFER_SIZE.getInt();
         http_config.setSecurePort(port);
         http_config.setRequestHeaderSize(bufferSize);
         http_config.setResponseHeaderSize(bufferSize);


Mime
View raw message