atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shweth...@apache.org
Subject [1/2] incubator-atlas git commit: ATLAS-37 atlas repository, webapp, hive-bridge tests fails with Hbase and Solr as Titan storage backend (suma.shivaprasad via shwethags)
Date Wed, 22 Jul 2015 08:52:12 GMT
Repository: incubator-atlas
Updated Branches:
  refs/heads/master b832faf0c -> 266d7cc00


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java b/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java
index 174dafb..ca71293 100755
--- a/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java
+++ b/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java
@@ -23,8 +23,14 @@ import org.apache.commons.configuration.PropertiesConfiguration;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.alias.CredentialProvider;
 import org.apache.hadoop.security.alias.CredentialProviderFactory;
-import org.mortbay.jetty.Connector;
-import org.mortbay.jetty.security.SslSocketConnector;
+import org.eclipse.jetty.server.Connector;
+import org.eclipse.jetty.server.HttpConfiguration;
+import org.eclipse.jetty.server.HttpConnectionFactory;
+import org.eclipse.jetty.server.SecureRequestCustomizer;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.server.SslConnectionFactory;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.eclipse.jetty.http.HttpVersion;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -54,18 +60,38 @@ public class SecureEmbeddedServer extends EmbeddedServer {
     protected Connector getConnector(int port) throws IOException {
         PropertiesConfiguration config = getConfiguration();
 
-        SslSocketConnector connector = new SslSocketConnector();
-        connector.setPort(port);
-        connector.setHost("0.0.0.0");
-        connector.setKeystore(config.getString(KEYSTORE_FILE_KEY,
-                System.getProperty(KEYSTORE_FILE_KEY, DEFAULT_KEYSTORE_FILE_LOCATION)));
-        connector.setKeyPassword(getPassword(config, SERVER_CERT_PASSWORD_KEY));
-        connector.setTruststore(config.getString(TRUSTSTORE_FILE_KEY,
-                System.getProperty(TRUSTSTORE_FILE_KEY, DEFATULT_TRUSTORE_FILE_LOCATION)));
-        connector.setTrustPassword(getPassword(config, TRUSTSTORE_PASSWORD_KEY));
-        connector.setPassword(getPassword(config, KEYSTORE_PASSWORD_KEY));
-        connector.setWantClientAuth(config.getBoolean(CLIENT_AUTH_KEY, Boolean.getBoolean(CLIENT_AUTH_KEY)));
-        return connector;
+        SslContextFactory sslContextFactory = new SslContextFactory();
+        sslContextFactory.setKeyStorePath(config.getString(KEYSTORE_FILE_KEY,
+            System.getProperty(KEYSTORE_FILE_KEY, DEFAULT_KEYSTORE_FILE_LOCATION)));
+        sslContextFactory.setKeyStorePassword(getPassword(config, KEYSTORE_PASSWORD_KEY));
+        sslContextFactory.setKeyManagerPassword(getPassword(config, SERVER_CERT_PASSWORD_KEY));
+        sslContextFactory.setTrustStorePath(config.getString(TRUSTSTORE_FILE_KEY,
+            System.getProperty(TRUSTSTORE_FILE_KEY, DEFATULT_TRUSTORE_FILE_LOCATION)));
+        sslContextFactory.setTrustStorePassword(getPassword(config, TRUSTSTORE_PASSWORD_KEY));
+        sslContextFactory.setWantClientAuth(config.getBoolean(CLIENT_AUTH_KEY, Boolean.getBoolean(CLIENT_AUTH_KEY)));
+
+        // SSL HTTP Configuration
+        // HTTP Configuration
+        HttpConfiguration http_config = new HttpConfiguration();
+        http_config.setSecureScheme("https");
+        final int bufferSize = getBufferSize();
+        http_config.setSecurePort(port);
+        http_config.setRequestHeaderSize(bufferSize);
+        http_config.setResponseHeaderSize(bufferSize);
+        http_config.setSendServerVersion(true);
+        http_config.setSendDateHeader(false);
+
+        HttpConfiguration https_config = new HttpConfiguration(http_config);
+        https_config.addCustomizer(new SecureRequestCustomizer());
+
+        // SSL Connector
+        ServerConnector sslConnector = new ServerConnector(server,
+            new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()),
+            new HttpConnectionFactory(https_config));
+        sslConnector.setPort(port);
+        server.addConnector(sslConnector);
+
+        return sslConnector;
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/main/resources/application.properties
----------------------------------------------------------------------
diff --git a/webapp/src/main/resources/application.properties b/webapp/src/main/resources/application.properties
index c9b8408..ecfdc38 100755
--- a/webapp/src/main/resources/application.properties
+++ b/webapp/src/main/resources/application.properties
@@ -18,16 +18,29 @@
 
 #########  Graph Database Configs  #########
 # Graph Storage
-atlas.graph.storage.backend=berkeleyje
+atlas.graph.storage.backend=${titan.storage.backend}
+
+# Graph Search Index Backend
+atlas.graph.index.search.backend=${titan.index.backend}
+
+#Berkeley storage directory
 atlas.graph.storage.directory=target/data/berkley
 
-# Graph Search Index
-atlas.graph.index.search.backend=elasticsearch
+#hbase
+#For standalone mode , specify localhost
+#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
+atlas.graph.storage.hostname=${titan.storage.hostname}
+
+#ElasticSearch
 atlas.graph.index.search.directory=target/data/es
 atlas.graph.index.search.elasticsearch.client-only=false
 atlas.graph.index.search.elasticsearch.local-mode=true
 atlas.graph.index.search.elasticsearch.create.sleep=2000
 
+# Solr cloud mode properties
+atlas.graph.index.search.solr.mode=cloud
+atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
+
 #########  Hive Lineage Configs  #########
 # This models reflects the base super types for Data and Process
 #atlas.lineage.hive.table.type.name=DataSet

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationKerberosFilterIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationKerberosFilterIT.java
b/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationKerberosFilterIT.java
index 95acc57..bb42dcf 100644
--- a/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationKerberosFilterIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationKerberosFilterIT.java
@@ -16,12 +16,12 @@
  */
 package org.apache.atlas.web.filters;
 
-import org.apache.atlas.security.BaseSecurityTest;
+import org.apache.atlas.web.security.BaseSecurityTest;
 import org.apache.atlas.web.service.EmbeddedServer;
 import org.apache.commons.configuration.ConfigurationException;
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.hdfs.web.URLConnectionFactory;
-import org.mortbay.jetty.Server;
+import org.eclipse.jetty.server.Server;
 import org.testng.Assert;
 import org.testng.annotations.Test;
 

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationSimpleFilterIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationSimpleFilterIT.java
b/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationSimpleFilterIT.java
index 5ea2e15..477ac4e 100644
--- a/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationSimpleFilterIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationSimpleFilterIT.java
@@ -16,10 +16,10 @@
  */
 package org.apache.atlas.web.filters;
 
-import org.apache.atlas.security.BaseSecurityTest;
+import org.apache.atlas.web.security.BaseSecurityTest;
 import org.apache.atlas.web.service.EmbeddedServer;
 import org.apache.commons.configuration.ConfigurationException;
-import org.mortbay.jetty.Server;
+import org.eclipse.jetty.server.Server;
 import org.testng.Assert;
 import org.testng.annotations.Test;
 

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/test/java/org/apache/atlas/web/listeners/LoginProcessorIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/listeners/LoginProcessorIT.java b/webapp/src/test/java/org/apache/atlas/web/listeners/LoginProcessorIT.java
index 54a7a7b..2148fe2 100644
--- a/webapp/src/test/java/org/apache/atlas/web/listeners/LoginProcessorIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/listeners/LoginProcessorIT.java
@@ -16,7 +16,7 @@
  */
 package org.apache.atlas.web.listeners;
 
-import org.apache.atlas.security.BaseSecurityTest;
+import org.apache.atlas.web.security.BaseSecurityTest;
 import org.apache.commons.configuration.ConfigurationException;
 import org.apache.commons.configuration.PropertiesConfiguration;
 import org.apache.hadoop.conf.Configuration;

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/test/java/org/apache/atlas/web/listeners/TestGuiceServletConfig.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/listeners/TestGuiceServletConfig.java
b/webapp/src/test/java/org/apache/atlas/web/listeners/TestGuiceServletConfig.java
new file mode 100644
index 0000000..3ba6387
--- /dev/null
+++ b/webapp/src/test/java/org/apache/atlas/web/listeners/TestGuiceServletConfig.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.web.listeners;
+
+import com.google.inject.Key;
+import com.google.inject.Provider;
+import com.google.inject.TypeLiteral;
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.thinkaurelius.titan.core.util.TitanCleanup;
+import com.tinkerpop.blueprints.Graph;
+import org.apache.atlas.repository.graph.GraphProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.ServletContextEvent;
+
+public class TestGuiceServletConfig extends GuiceServletConfig {
+
+    private static final Logger LOG = LoggerFactory.getLogger(TestGuiceServletConfig.class);
+
+    @Override
+    public void contextInitialized(ServletContextEvent servletContextEvent) {
+        LOG.info("Initializing test servlet listener");
+        super.contextInitialized(servletContextEvent);
+    }
+
+    @Override
+    public void contextDestroyed(ServletContextEvent servletContextEvent) {
+        if(injector != null) {
+            TypeLiteral<GraphProvider<TitanGraph>> graphProviderType = new TypeLiteral<GraphProvider<TitanGraph>>()
{};
+            Provider<GraphProvider<TitanGraph>> graphProvider = injector.getProvider(Key.get(graphProviderType));
+            TitanGraph graph = graphProvider.get().get();
+
+            LOG.info("Clearing graph store");
+            try {
+                graph.shutdown();
+                TitanCleanup.clear(graph);
+            } catch (Exception e) {
+                LOG.warn("Clearing graph store failed ", e);
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/test/java/org/apache/atlas/web/security/BaseSecurityTest.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/security/BaseSecurityTest.java b/webapp/src/test/java/org/apache/atlas/web/security/BaseSecurityTest.java
new file mode 100644
index 0000000..67f3901
--- /dev/null
+++ b/webapp/src/test/java/org/apache/atlas/web/security/BaseSecurityTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.web.security;
+
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.zookeeper.Environment;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.webapp.WebAppContext;
+import org.testng.Assert;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.Writer;
+import java.nio.file.Files;
+import java.util.Locale;
+import java.util.Properties;
+
+/**
+ *
+ */
+public class BaseSecurityTest {
+    private static final String JAAS_ENTRY = "%s { \n" + " %s required\n"
+            // kerberos module
+            + " keyTab=\"%s\"\n" + " debug=true\n" + " principal=\"%s\"\n" + " useKeyTab=true\n"
+            + " useTicketCache=false\n" + " doNotPrompt=true\n" + " storeKey=true;\n" + "};
\n";
+    protected MiniKdc kdc;
+
+    protected String getWarPath() {
+        return String.format("/target/atlas-webapp-%s.war",
+                System.getProperty("release.version"));
+    }
+
+    protected void generateTestProperties(Properties props) throws ConfigurationException,
IOException {
+        PropertiesConfiguration config =
+                new PropertiesConfiguration(System.getProperty("user.dir") + "/../src/conf/application.properties");
+        for (String propName : props.stringPropertyNames()) {
+            config.setProperty(propName, props.getProperty(propName));
+        }
+        File file = new File(System.getProperty("user.dir"), "application.properties");
+        file.deleteOnExit();
+        Writer fileWriter = new FileWriter(file);
+        config.save(fileWriter);
+    }
+
+    protected void startEmbeddedServer(Server server) throws Exception {
+        WebAppContext webapp = new WebAppContext();
+        webapp.setContextPath("/");
+        webapp.setWar(System.getProperty("user.dir") + getWarPath());
+        server.setHandler(webapp);
+
+        server.start();
+    }
+
+    protected File startKDC() throws Exception {
+        File target = Files.createTempDirectory("sectest").toFile();
+        File kdcWorkDir = new File(target, "kdc");
+        Properties kdcConf = MiniKdc.createConf();
+        kdcConf.setProperty(MiniKdc.DEBUG, "true");
+        kdc = new MiniKdc(kdcConf, kdcWorkDir);
+        kdc.start();
+
+        Assert.assertNotNull(kdc.getRealm());
+        return kdcWorkDir;
+    }
+
+    public String createJAASEntry(String context, String principal, File keytab) {
+        String keytabpath = keytab.getAbsolutePath();
+        // fix up for windows; no-op on unix
+        keytabpath = keytabpath.replace('\\', '/');
+        return String.format(Locale.ENGLISH, JAAS_ENTRY, context, getKerberosAuthModuleForJVM(),
keytabpath, principal);
+    }
+
+    protected String getKerberosAuthModuleForJVM() {
+        if (System.getProperty("java.vendor").contains("IBM")) {
+            return "com.ibm.security.auth.module.Krb5LoginModule";
+        } else {
+            return "com.sun.security.auth.module.Krb5LoginModule";
+        }
+    }
+
+    protected void bindJVMtoJAASFile(File jaasFile) {
+        String path = jaasFile.getAbsolutePath();
+        System.setProperty(Environment.JAAS_CONF_KEY, path);
+    }
+
+    protected File createKeytab(MiniKdc kdc, File kdcWorkDir, String principal, String filename)
throws Exception {
+        File keytab = new File(kdcWorkDir, filename);
+        kdc.createPrincipal(keytab, principal, principal + "/localhost", principal + "/127.0.0.1");
+        return keytab;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerIT.java
b/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerIT.java
index 3e27532..7eb36d8 100644
--- a/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerIT.java
@@ -38,6 +38,7 @@ public class SecureEmbeddedServerIT extends SecureEmbeddedServerITBase {
         SecureEmbeddedServer secureEmbeddedServer = null;
         try {
             String appPath = System.getProperty("user.dir") + getWarPath();
+
             secureEmbeddedServer = new SecureEmbeddedServer(21443, appPath) {
                 @Override
                 protected PropertiesConfiguration getConfiguration() {
@@ -53,6 +54,8 @@ public class SecureEmbeddedServerIT extends SecureEmbeddedServerITBase {
 
             // test to see whether server is up and root page can be served
             Assert.assertEquals(connection.getResponseCode(), 200);
+        } catch(Throwable e) {
+            Assert.fail("War deploy failed", e);
         } finally {
             secureEmbeddedServer.server.stop();
         }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerITBase.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerITBase.java
b/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerITBase.java
index db110ca..9a5b8ad 100755
--- a/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerITBase.java
+++ b/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerITBase.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.security.alias.CredentialProvider;
 import org.apache.hadoop.security.alias.CredentialProviderFactory;
 import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
-import org.mortbay.jetty.webapp.WebAppContext;
+import org.eclipse.jetty.webapp.WebAppContext;
 import org.testng.Assert;
 import org.testng.TestListenerAdapter;
 import org.testng.TestNG;

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/test/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/webapp/src/test/webapp/WEB-INF/web.xml b/webapp/src/test/webapp/WEB-INF/web.xml
new file mode 100755
index 0000000..0562643
--- /dev/null
+++ b/webapp/src/test/webapp/WEB-INF/web.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<!DOCTYPE web-app PUBLIC "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN"
+        "http://java.sun.com/dtd/web-app_2_3.dtd">
+
+<web-app>
+    <display-name>Apache Atlas</display-name>
+    <description>Metadata Management and Data Governance Platform over Hadoop</description>
+
+    <context-param>
+        <param-name>guice.packages</param-name>
+        <param-value>
+            org.apache.atlas.web.resources,org.apache.atlas.web.params
+        </param-value>
+    </context-param>
+
+    <!--
+        More information can be found here:
+
+        https://jersey.java.net/nonav/apidocs/1.11/contribs/jersey-guice/com/sun/jersey/guice/spi/container/servlet/package-summary.html
+     -->
+
+    <filter>
+        <filter-name>guiceFilter</filter-name>
+        <filter-class>com.google.inject.servlet.GuiceFilter</filter-class>
+    </filter>
+
+    <filter-mapping>
+        <filter-name>guiceFilter</filter-name>
+        <url-pattern>/*</url-pattern>
+    </filter-mapping>
+
+    <listener>
+        <listener-class>org.apache.atlas.web.listeners.TestGuiceServletConfig</listener-class>
+    </listener>
+</web-app>


Mime
View raw message