Return-Path: X-Original-To: apmail-atlas-commits-archive@minotaur.apache.org Delivered-To: apmail-atlas-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id C9BCD1841F for ; Wed, 22 Jul 2015 08:52:41 +0000 (UTC) Received: (qmail 23019 invoked by uid 500); 22 Jul 2015 08:52:29 -0000 Delivered-To: apmail-atlas-commits-archive@atlas.apache.org Received: (qmail 22987 invoked by uid 500); 22 Jul 2015 08:52:29 -0000 Mailing-List: contact commits-help@atlas.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@atlas.incubator.apache.org Delivered-To: mailing list commits@atlas.incubator.apache.org Received: (qmail 22977 invoked by uid 99); 22 Jul 2015 08:52:28 -0000 Received: from Unknown (HELO spamd1-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 22 Jul 2015 08:52:28 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd1-us-west.apache.org (ASF Mail Server at spamd1-us-west.apache.org) with ESMTP id 6214BD6A1C for ; Wed, 22 Jul 2015 08:52:28 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd1-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: 0.574 X-Spam-Level: X-Spam-Status: No, score=0.574 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RP_MATCHES_RCVD=-1.227, URIBL_BLOCKED=0.001] autolearn=disabled Received: from mx1-us-west.apache.org ([10.40.0.8]) by localhost (spamd1-us-west.apache.org [10.40.0.7]) (amavisd-new, port 10024) with ESMTP id ftYeolLVFcG6 for ; Wed, 22 Jul 2015 08:52:13 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-us-west.apache.org (ASF Mail Server at mx1-us-west.apache.org) with SMTP id 3C2902122A for ; Wed, 22 Jul 2015 08:52:13 +0000 (UTC) Received: (qmail 22784 invoked by uid 99); 22 Jul 2015 08:52:13 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 22 Jul 2015 08:52:13 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D4BB6DFF81; Wed, 22 Jul 2015 08:52:12 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: shwethags@apache.org To: commits@atlas.incubator.apache.org Date: Wed, 22 Jul 2015 08:52:13 -0000 Message-Id: In-Reply-To: <3926c33720b3424eb347d2d11fe8b3bd@git.apache.org> References: <3926c33720b3424eb347d2d11fe8b3bd@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/2] incubator-atlas git commit: ATLAS-37 atlas repository, webapp, hive-bridge tests fails with Hbase and Solr as Titan storage backend (suma.shivaprasad via shwethags) ATLAS-37 atlas repository, webapp, hive-bridge tests fails with Hbase and Solr as Titan storage backend (suma.shivaprasad via shwethags) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/266d7cc0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/266d7cc0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/266d7cc0 Branch: refs/heads/master Commit: 266d7cc00bef49356bcdb1290930e05748182b12 Parents: b832faf Author: Shwetha GS Authored: Wed Jul 22 14:22:03 2015 +0530 Committer: Shwetha GS Committed: Wed Jul 22 14:22:03 2015 +0530 ---------------------------------------------------------------------- addons/hive-bridge/pom.xml | 57 +++-- .../atlas/hive/hook/BaseSSLAndKerberosTest.java | 5 +- .../hook/NegativeSSLAndKerberosHiveHookIT.java | 2 +- .../hive/hook/SSLAndKerberosHiveHookIT.java | 2 +- .../apache/atlas/hive/hook/SSLHiveHookIT.java | 4 +- .../src/test/resources/application.properties | 34 ++- .../apache/atlas/security/BaseSecurityTest.java | 108 --------- pom.xml | 191 ++++++++++++++-- release-log.txt | 1 + repository/pom.xml | 16 ++ .../apache/atlas/RepositoryMetadataModule.java | 1 + .../atlas/repository/graph/GraphProvider.java | 1 - .../repository/graph/TitanGraphProvider.java | 28 ++- .../atlas/services/DefaultMetadataService.java | 3 +- .../atlas/RepositoryServiceLoadingTest.java | 3 + .../GraphBackedDiscoveryServiceTest.java | 7 + .../atlas/discovery/HiveLineageServiceTest.java | 223 ++++++++++--------- .../GraphBackedMetadataRepositoryTest.java | 28 ++- .../typestore/GraphBackedTypeStoreTest.java | 13 ++ .../src/test/resources/application.properties | 26 ++- src/conf/application.properties | 17 ++ webapp/pom.xml | 139 ++++++------ .../atlas/web/listeners/GuiceServletConfig.java | 15 +- .../atlas/web/service/EmbeddedServer.java | 28 ++- .../atlas/web/service/SecureEmbeddedServer.java | 54 +++-- .../src/main/resources/application.properties | 19 +- .../MetadataAuthenticationKerberosFilterIT.java | 4 +- .../MetadataAuthenticationSimpleFilterIT.java | 4 +- .../atlas/web/listeners/LoginProcessorIT.java | 2 +- .../web/listeners/TestGuiceServletConfig.java | 57 +++++ .../atlas/web/security/BaseSecurityTest.java | 108 +++++++++ .../web/service/SecureEmbeddedServerIT.java | 3 + .../web/service/SecureEmbeddedServerITBase.java | 2 +- webapp/src/test/webapp/WEB-INF/web.xml | 53 +++++ 34 files changed, 885 insertions(+), 373 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/addons/hive-bridge/pom.xml ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/pom.xml b/addons/hive-bridge/pom.xml index 2617aeb..117d76c 100755 --- a/addons/hive-bridge/pom.xml +++ b/addons/hive-bridge/pom.xml @@ -59,6 +59,13 @@ hive-metastore ${hive.version} provided + + + org.mortbay.jetty + * + + + @@ -73,6 +80,17 @@ hive-cli ${hive.version} test + + + javax.servlet + * + + + org.eclipse.jetty.aggregate + * + + + @@ -116,8 +134,16 @@ - org.mortbay.jetty - jetty + org.apache.atlas + atlas-webapp + test-jar + ${project.version} + test + + + + org.eclipse.jetty + jetty-server test @@ -229,21 +255,22 @@ - org.mortbay.jetty - maven-jetty-plugin - ${jetty.version} + org.eclipse.jetty + jetty-maven-plugin ${skipTests} - - - 21000 - 60000 - - - ../../webapp/target/atlas-webapp-${project.version}.war - / - true + + 21000 + 60000 + + ../../webapp/target/atlas-webapp-${project.version}.war + true + + / + ../../webapp/src/test/webapp/WEB-INF/web.xml + + true atlas.log.dir @@ -251,7 +278,7 @@ atlas.conf - addons/hive-bridge/src/test/resources + ${project.build.directory}/test-classes atlas-stop http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java index 5ae6fe2..11163c8 100644 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java @@ -16,8 +16,7 @@ */ package org.apache.atlas.hive.hook; -import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; -import org.apache.atlas.security.BaseSecurityTest; +import org.apache.atlas.web.security.BaseSecurityTest; import org.apache.atlas.web.service.SecureEmbeddedServer; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.io.FileUtils; @@ -26,7 +25,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProviderFactory; -import org.mortbay.jetty.Server; +import org.eclipse.jetty.server.Server; import java.io.File; import java.io.IOException; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/NegativeSSLAndKerberosHiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/NegativeSSLAndKerberosHiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/NegativeSSLAndKerberosHiveHookIT.java index 629a9f3..072c36b 100755 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/NegativeSSLAndKerberosHiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/NegativeSSLAndKerberosHiveHookIT.java @@ -29,7 +29,7 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.security.alias.JavaKeyStoreProvider; import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.security.ssl.SSLHostnameVerifier; -import org.mortbay.jetty.webapp.WebAppContext; +import org.eclipse.jetty.webapp.WebAppContext; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLAndKerberosHiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLAndKerberosHiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLAndKerberosHiveHookIT.java index b8f29aa..6fab2ee 100755 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLAndKerberosHiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLAndKerberosHiveHookIT.java @@ -34,7 +34,7 @@ import org.apache.hadoop.security.alias.JavaKeyStoreProvider; import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.security.ssl.SSLHostnameVerifier; import org.codehaus.jettison.json.JSONArray; -import org.mortbay.jetty.webapp.WebAppContext; +import org.eclipse.jetty.webapp.WebAppContext; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java index 54bfaf6..b114d5a 100755 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java @@ -37,8 +37,8 @@ import org.apache.hadoop.security.alias.JavaKeyStoreProvider; import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.security.ssl.SSLHostnameVerifier; import org.codehaus.jettison.json.JSONArray; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.webapp.WebAppContext; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.webapp.WebAppContext; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/addons/hive-bridge/src/test/resources/application.properties ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/resources/application.properties b/addons/hive-bridge/src/test/resources/application.properties index 50eefed..dda9a18 100644 --- a/addons/hive-bridge/src/test/resources/application.properties +++ b/addons/hive-bridge/src/test/resources/application.properties @@ -17,23 +17,45 @@ # ######### Graph Database Configs ######### +#Refer http://s3.thinkaurelius.com/docs/titan/0.5.1/titan-config-ref.html # Graph Storage -atlas.graph.storage.backend=inmemory +atlas.graph.storage.backend=${titan.storage.backend} -# Graph Search Index -atlas.graph.index.search.backend=lucene -atlas.graph.index.search.directory=target/data/lucene +#Berkeley storage directory +atlas.graph.storage.directory=target/data/berkley +#hbase +#For standalone mode , specify localhost +#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2 +atlas.graph.storage.hostname=${titan.storage.hostname} + +# Graph Search Index Backend +atlas.graph.index.search.backend=${titan.index.backend} + +#lucene +#atlas.graph.index.search.directory=target/data/lucene + +#elasticsearch +atlas.graph.index.search.directory=./target/data/es +atlas.graph.index.search.elasticsearch.client-only=false +atlas.graph.index.search.elasticsearch.local-mode=true +atlas.graph.index.search.elasticsearch.create.sleep=2000 + +#solr in cloud mode +atlas.graph.index.search.solr.mode=cloud +atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address} + +#solr in http mode +atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr ######### Hive Lineage Configs ######### -# This models reflects the base super types for Data and Process #atlas.lineage.hive.table.type.name=DataSet #atlas.lineage.hive.process.type.name=Process #atlas.lineage.hive.process.inputs.name=inputs #atlas.lineage.hive.process.outputs.name=outputs ## Schema -#atlas.lineage.hive.table.schema.query=hive_table where name=?, columns +#atlas.lineage.hive.table.schema.query.hive_table=hive_table where name='%s'\, columns ######### Security Properties ######### http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/client/src/test/java/org/apache/atlas/security/BaseSecurityTest.java ---------------------------------------------------------------------- diff --git a/client/src/test/java/org/apache/atlas/security/BaseSecurityTest.java b/client/src/test/java/org/apache/atlas/security/BaseSecurityTest.java deleted file mode 100644 index 598f56b..0000000 --- a/client/src/test/java/org/apache/atlas/security/BaseSecurityTest.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.security; - -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.hadoop.minikdc.MiniKdc; -import org.apache.zookeeper.Environment; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.webapp.WebAppContext; -import org.testng.Assert; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.Writer; -import java.nio.file.Files; -import java.util.Locale; -import java.util.Properties; - -/** - * - */ -public class BaseSecurityTest { - private static final String JAAS_ENTRY = "%s { \n" + " %s required\n" - // kerberos module - + " keyTab=\"%s\"\n" + " debug=true\n" + " principal=\"%s\"\n" + " useKeyTab=true\n" - + " useTicketCache=false\n" + " doNotPrompt=true\n" + " storeKey=true;\n" + "}; \n"; - protected MiniKdc kdc; - - protected String getWarPath() { - return String.format("/target/atlas-webapp-%s.war", - System.getProperty("release.version")); - } - - protected void generateTestProperties(Properties props) throws ConfigurationException, IOException { - PropertiesConfiguration config = - new PropertiesConfiguration(System.getProperty("user.dir") + "/../src/conf/application.properties"); - for (String propName : props.stringPropertyNames()) { - config.setProperty(propName, props.getProperty(propName)); - } - File file = new File(System.getProperty("user.dir"), "application.properties"); - file.deleteOnExit(); - Writer fileWriter = new FileWriter(file); - config.save(fileWriter); - } - - protected void startEmbeddedServer(Server server) throws Exception { - WebAppContext webapp = new WebAppContext(); - webapp.setContextPath("/"); - webapp.setWar(System.getProperty("user.dir") + getWarPath()); - server.setHandler(webapp); - - server.start(); - } - - protected File startKDC() throws Exception { - File target = Files.createTempDirectory("sectest").toFile(); - File kdcWorkDir = new File(target, "kdc"); - Properties kdcConf = MiniKdc.createConf(); - kdcConf.setProperty(MiniKdc.DEBUG, "true"); - kdc = new MiniKdc(kdcConf, kdcWorkDir); - kdc.start(); - - Assert.assertNotNull(kdc.getRealm()); - return kdcWorkDir; - } - - public String createJAASEntry(String context, String principal, File keytab) { - String keytabpath = keytab.getAbsolutePath(); - // fix up for windows; no-op on unix - keytabpath = keytabpath.replace('\\', '/'); - return String.format(Locale.ENGLISH, JAAS_ENTRY, context, getKerberosAuthModuleForJVM(), keytabpath, principal); - } - - protected String getKerberosAuthModuleForJVM() { - if (System.getProperty("java.vendor").contains("IBM")) { - return "com.ibm.security.auth.module.Krb5LoginModule"; - } else { - return "com.sun.security.auth.module.Krb5LoginModule"; - } - } - - protected void bindJVMtoJAASFile(File jaasFile) { - String path = jaasFile.getAbsolutePath(); - System.setProperty(Environment.JAAS_CONF_KEY, path); - } - - protected File createKeytab(MiniKdc kdc, File kdcWorkDir, String principal, String filename) throws Exception { - File keytab = new File(kdcWorkDir, filename); - kdc.createPrincipal(keytab, principal, principal + "/localhost", principal + "/127.0.0.1"); - return keytab; - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index b94c92b..a8afcc3 100755 --- a/pom.xml +++ b/pom.xml @@ -322,11 +322,13 @@ UTF-8 1.7.7 - 6.1.26 + 9.2.12.v20150709 1.9 + 1.8.3 2.6.0 0.5.4 2.6.0 + 0.98.9-hadoop2 2.10.4 @@ -351,6 +353,8 @@ false + berkeleyje + elasticsearch @@ -381,8 +385,22 @@ - + + + distributed + + false + + + hbase + solr + localhost:9983 + localhost + + + + typesystem client @@ -455,9 +473,15 @@ - org.mortbay.jetty - jsp-2.1 - 6.0.0 + org.eclipse.jetty + jetty-jsp + ${jetty.version} + + + javax.servlet + * + + @@ -507,25 +531,68 @@ org.apache.hadoop hadoop-common ${hadoop.version} + + + javax.servlet + * + + + tomcat + * + + + org.mortbay.jetty + * + + + org.codehaus.jackson + * + + + org.htrace + * + + org.apache.hadoop - hadoop-client + hadoop-hdfs ${hadoop.version} + test + + + javax.servlet + * + + + tomcat + * + + + org.mortbay.jetty + * + + + org.apache.hadoop - hadoop-annotations + hadoop-client ${hadoop.version} + + + org.mortbay.jetty + * + + org.apache.hadoop - hadoop-hdfs + hadoop-annotations ${hadoop.version} - test @@ -615,6 +682,12 @@ com.sun.jersey jersey-json ${jersey.version} + + + javax.xml.stream + * + + @@ -639,19 +712,19 @@ org.codehaus.jackson jackson-core-asl - 1.5.2 + ${jackson.version} org.codehaus.jackson jackson-mapper-asl - 1.5.2 + ${jackson.version} org.codehaus.jettison jettison - 1.3 + 1.3.7 @@ -668,15 +741,15 @@ - org.mortbay.jetty - jetty + org.eclipse.jetty + jetty-server ${jetty.version} compile - org.mortbay.jetty - jetty-plus + org.eclipse.jetty + jetty-webapp ${jetty.version} compile @@ -736,12 +809,80 @@ com.thinkaurelius.titan + titan-hbase + ${titan.version} + + + + org.apache.hbase + hbase-client + ${hbase.version} + + + avro + org.apache.avro + + + jruby-complete + org.jruby + + + asm + asm + + + * + org.apache.hadoop + + + * + org.mortbay.jetty + + + + + + com.thinkaurelius.titan titan-es ${titan.version} com.thinkaurelius.titan + titan-solr + ${titan.version} + + + * + org.eclipse.jetty + + + * + javax.servlet + + + * + org.jruby.joni + + + * + org.eclipse.jetty.orbit + + + + * + org.restlet.jee + + + * + org.ow2.asm + + + + + + + com.thinkaurelius.titan titan-lucene ${titan.version} @@ -982,6 +1123,7 @@ src/test/resources + true @@ -1072,6 +1214,15 @@ + org.apache.maven.plugins + maven-resources-plugin + 2.7 + + UTF-8 + + + + net.alchim31.maven scala-maven-plugin 3.2.0 @@ -1222,12 +1373,12 @@ - org.mortbay.jetty - maven-jetty-plugin - 6.1.16 + org.eclipse.jetty + jetty-maven-plugin + ${jetty.version} - + true http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index d655478..76b8778 100644 --- a/release-log.txt +++ b/release-log.txt @@ -8,6 +8,7 @@ ATLAS-54 Rename configs in hive hook (shwethags) ATLAS-3 Mixed Index creation fails with Date types (suma.shivaprasad via shwethags) ALL CHANGES: +ATLAS-37 atlas repository, webapp, hive-bridge tests fails with Hbase and Solr as Titan storage backend (suma.shivaprasad via shwethags) ATLAS-56 atlas_config.py should give an informative error if jar or java binaries can't be found (dossett@gmail.com via shwethags) ATLAS-45 Entity submit fails (suma.shivaprasad via shwethags) ATLAS-46 Different data directory with restart (shwethags) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/repository/pom.xml ---------------------------------------------------------------------- diff --git a/repository/pom.xml b/repository/pom.xml index b23c289..03aa792 100755 --- a/repository/pom.xml +++ b/repository/pom.xml @@ -101,11 +101,26 @@ com.thinkaurelius.titan + titan-solr + + + + com.thinkaurelius.titan titan-berkeleyje com.thinkaurelius.titan + titan-hbase + + + + org.apache.hbase + hbase-client + + + + com.thinkaurelius.titan titan-lucene @@ -172,6 +187,7 @@ + org.apache.maven.plugins http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java index e7353f9..fbd01de 100755 --- a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java +++ b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java @@ -22,6 +22,7 @@ import com.google.inject.matcher.Matchers; import com.google.inject.multibindings.Multibinder; import com.google.inject.throwingproviders.ThrowingProviderBinder; import com.thinkaurelius.titan.core.TitanGraph; +import com.tinkerpop.blueprints.Graph; import org.aopalliance.intercept.MethodInterceptor; import org.apache.atlas.discovery.DiscoveryService; import org.apache.atlas.discovery.HiveLineageService; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/repository/src/main/java/org/apache/atlas/repository/graph/GraphProvider.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphProvider.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphProvider.java index 5472081..f89bdf5 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphProvider.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphProvider.java @@ -22,7 +22,6 @@ import com.google.inject.throwingproviders.CheckedProvider; import com.tinkerpop.blueprints.Graph; public interface GraphProvider extends CheckedProvider { - @Override T get(); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java b/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java index 3c1d151..d7ed1f1 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java @@ -30,6 +30,7 @@ import org.slf4j.LoggerFactory; import javax.inject.Singleton; import java.util.Iterator; +import java.util.Properties; /** * Default implementation for Graph Provider that doles out Titan Graph. @@ -43,11 +44,17 @@ public class TitanGraphProvider implements GraphProvider { */ private static final String ATLAS_PREFIX = "atlas.graph."; + private static TitanGraph graphInstance; + private static Configuration getConfiguration() throws AtlasException { PropertiesConfiguration configProperties = PropertiesUtil.getApplicationProperties(); Configuration graphConfig = new PropertiesConfiguration(); + Properties sysProperties = System.getProperties(); + LOG.info("System properties: "); + LOG.info(sysProperties.toString()); + final Iterator iterator = configProperties.getKeys(); while (iterator.hasNext()) { String key = iterator.next(); @@ -66,13 +73,20 @@ public class TitanGraphProvider implements GraphProvider { @Singleton @Provides public TitanGraph get() { - Configuration config; - try { - config = getConfiguration(); - } catch (AtlasException e) { - throw new RuntimeException(e); - } + if(graphInstance == null) { + synchronized (TitanGraphProvider.class) { + if(graphInstance == null) { + Configuration config; + try { + config = getConfiguration(); + } catch (AtlasException e) { + throw new RuntimeException(e); + } - return TitanFactory.open(config); + graphInstance = TitanFactory.open(config); + } + } + } + return graphInstance; } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java index 9556272..56168db 100755 --- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java +++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java @@ -81,7 +81,8 @@ public class DefaultMetadataService implements MetadataService { @Inject DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore, - final Collection> typeChangeListeners) throws AtlasException { + final Collection> typeChangeListeners) throws AtlasException { + this.typeStore = typeStore; this.typeSystem = TypeSystem.getInstance(); this.repository = repository; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java b/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java index 4195955..18b1d1b 100755 --- a/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java +++ b/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java @@ -19,8 +19,11 @@ package org.apache.atlas; import com.thinkaurelius.titan.core.TitanGraph; +import com.thinkaurelius.titan.core.util.TitanCleanup; import org.apache.atlas.repository.graph.GraphProvider; +import org.apache.atlas.typesystem.types.TypeSystem; import org.testng.Assert; +import org.testng.annotations.AfterClass; import org.testng.annotations.Guice; import org.testng.annotations.Test; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java index 67cdd78..127e6d7 100755 --- a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java @@ -20,6 +20,7 @@ package org.apache.atlas.discovery; import com.google.common.collect.ImmutableList; import com.thinkaurelius.titan.core.TitanGraph; +import com.thinkaurelius.titan.core.util.TitanCleanup; import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Vertex; import org.apache.atlas.RepositoryMetadataModule; @@ -117,6 +118,12 @@ public class GraphBackedDiscoveryServiceTest { @AfterClass public void tearDown() throws Exception { TypeSystem.getInstance().reset(); + graphProvider.get().shutdown(); + try { + TitanCleanup.clear(graphProvider.get()); + } catch(Exception e) { + e.printStackTrace(); + } } @Test http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java index ce00f8c..2c7d61a 100644 --- a/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,9 +20,12 @@ package org.apache.atlas.discovery; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.thinkaurelius.titan.core.TitanGraph; +import com.thinkaurelius.titan.core.util.TitanCleanup; import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; import org.apache.atlas.repository.EntityNotFoundException; +import org.apache.atlas.repository.graph.GraphProvider; import org.apache.atlas.services.DefaultMetadataService; import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.TypesDef; @@ -38,17 +41,22 @@ import org.apache.atlas.typesystem.types.IDataType; import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.StructTypeDefinition; import org.apache.atlas.typesystem.types.TraitType; +import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.TypeUtils; import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONObject; import org.testng.Assert; +import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Guice; import org.testng.annotations.Test; import javax.inject.Inject; +import java.io.File; +import java.io.IOException; +import java.util.HashSet; import java.util.List; /** @@ -58,65 +66,76 @@ import java.util.List; public class HiveLineageServiceTest { @Inject + private GraphBackedDiscoveryService discoveryService; + + @Inject private DefaultMetadataService metadataService; @Inject - private GraphBackedDiscoveryService discoveryService; + private GraphProvider graphProvider; @Inject private HiveLineageService hiveLineageService; - // @Inject - // private GraphProvider graphProvider; @BeforeClass public void setUp() throws Exception { setUpTypes(); setupInstances(); - // TestUtils.dumpGraph(graphProvider.get()); } + @AfterClass + public void tearDown() throws Exception { + TypeSystem.getInstance().reset(); + graphProvider.get().shutdown(); + try { + TitanCleanup.clear(graphProvider.get()); + } catch (Exception e) { + e.printStackTrace(); + } + } + @DataProvider(name = "dslQueriesProvider") private Object[][] createDSLQueries() { return new String[][]{ - // joins - {"hive_table where name=\"sales_fact\", columns"}, - {"hive_table where name=\"sales_fact\", columns select name, dataType, comment"}, - {"hive_table where name=\"sales_fact\", columns as c select c.name, c.dataType, c.comment"}, - // {"hive_db as db where (db.name=\"Reporting\"), hive_table as table select db.name, - // table.name"}, - {"from hive_db"}, {"hive_db"}, {"hive_db where hive_db.name=\"Reporting\""}, - {"hive_db hive_db.name = \"Reporting\""}, - {"hive_db where hive_db.name=\"Reporting\" select name, owner"}, {"hive_db has name"}, - // {"hive_db, hive_table"}, - // {"hive_db, hive_process has name"}, - // {"hive_db as db1, hive_table where db1.name = \"Reporting\""}, - // {"hive_db where hive_db.name=\"Reporting\" and hive_db.createTime < " + System - // .currentTimeMillis()}, - {"from hive_table"}, {"hive_table"}, {"hive_table is Dimension"}, - {"hive_column where hive_column isa PII"}, - // {"hive_column where hive_column isa PII select hive_column.name"}, - {"hive_column select hive_column.name"}, {"hive_column select name"}, - {"hive_column where hive_column.name=\"customer_id\""}, {"from hive_table select hive_table.name"}, - {"hive_db where (name = \"Reporting\")"}, - {"hive_db where (name = \"Reporting\") select name as _col_0, owner as _col_1"}, - {"hive_db where hive_db has name"}, - // {"hive_db hive_table"}, - {"hive_db where hive_db has name"}, - // {"hive_db as db1 hive_table where (db1.name = \"Reporting\")"}, - {"hive_db where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 "}, - // {"hive_db where (name = \"Reporting\") and ((createTime + 1) > 0)"}, - // {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name = - // \"Reporting\") select db1.name as dbName, tab.name as tabName"}, - // {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) or (db1.name = - // \"Reporting\") select db1.name as dbName, tab.name as tabName"}, - // {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name = - // \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"}, - // {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name = - // \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"}, - // trait searches - {"Dimension"}, {"Fact"}, {"ETL"}, {"Metric"}, {"PII"},}; + // joins + {"hive_table where name=\"sales_fact\", columns"}, + {"hive_table where name=\"sales_fact\", columns select name, dataType, comment"}, + {"hive_table where name=\"sales_fact\", columns as c select c.name, c.dataType, c.comment"}, + // {"hive_db as db where (db.name=\"Reporting\"), hive_table as table select db.name, + // table.name"}, + {"from hive_db"}, {"hive_db"}, {"hive_db where hive_db.name=\"Reporting\""}, + {"hive_db hive_db.name = \"Reporting\""}, + {"hive_db where hive_db.name=\"Reporting\" select name, owner"}, {"hive_db has name"}, + // {"hive_db, hive_table"}, + // {"hive_db, hive_process has name"}, + // {"hive_db as db1, hive_table where db1.name = \"Reporting\""}, + // {"hive_db where hive_db.name=\"Reporting\" and hive_db.createTime < " + System + // .currentTimeMillis()}, + {"from hive_table"}, {"hive_table"}, {"hive_table is Dimension"}, + {"hive_column where hive_column isa PII"}, + // {"hive_column where hive_column isa PII select hive_column.name"}, + {"hive_column select hive_column.name"}, {"hive_column select name"}, + {"hive_column where hive_column.name=\"customer_id\""}, {"from hive_table select hive_table.name"}, + {"hive_db where (name = \"Reporting\")"}, + {"hive_db where (name = \"Reporting\") select name as _col_0, owner as _col_1"}, + {"hive_db where hive_db has name"}, + // {"hive_db hive_table"}, + {"hive_db where hive_db has name"}, + // {"hive_db as db1 hive_table where (db1.name = \"Reporting\")"}, + {"hive_db where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 "}, + // {"hive_db where (name = \"Reporting\") and ((createTime + 1) > 0)"}, + // {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name = + // \"Reporting\") select db1.name as dbName, tab.name as tabName"}, + // {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) or (db1.name = + // \"Reporting\") select db1.name as dbName, tab.name as tabName"}, + // {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name = + // \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"}, + // {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name = + // \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"}, + // trait searches + {"Dimension"}, {"Fact"}, {"ETL"}, {"Metric"}, {"PII"},}; } @Test(dataProvider = "dslQueriesProvider") @@ -242,7 +261,7 @@ public class HiveLineageServiceTest { @DataProvider(name = "tableNamesProvider") private Object[][] tableNames() { return new String[][]{{"sales_fact", "4"}, {"time_dim", "3"}, {"sales_fact_daily_mv", "4"}, - {"sales_fact_monthly_mv", "4"}}; + {"sales_fact_monthly_mv", "4"}}; } @Test(dataProvider = "tableNamesProvider") @@ -296,45 +315,45 @@ public class HiveLineageServiceTest { private TypesDef createTypeDefinitions() { HierarchicalTypeDefinition dbClsDef = TypesUtil - .createClassTypeDef(DATABASE_TYPE, null, attrDef("name", DataTypes.STRING_TYPE), - attrDef("description", DataTypes.STRING_TYPE), attrDef("locationUri", DataTypes.STRING_TYPE), - attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE)); + .createClassTypeDef(DATABASE_TYPE, null, attrDef("name", DataTypes.STRING_TYPE), + attrDef("description", DataTypes.STRING_TYPE), attrDef("locationUri", DataTypes.STRING_TYPE), + attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE)); HierarchicalTypeDefinition storageDescClsDef = TypesUtil - .createClassTypeDef(STORAGE_DESC_TYPE, null, attrDef("location", DataTypes.STRING_TYPE), - attrDef("inputFormat", DataTypes.STRING_TYPE), attrDef("outputFormat", DataTypes.STRING_TYPE), - attrDef("compressed", DataTypes.STRING_TYPE, Multiplicity.REQUIRED, false, null)); + .createClassTypeDef(STORAGE_DESC_TYPE, null, attrDef("location", DataTypes.STRING_TYPE), + attrDef("inputFormat", DataTypes.STRING_TYPE), attrDef("outputFormat", DataTypes.STRING_TYPE), + attrDef("compressed", DataTypes.STRING_TYPE, Multiplicity.REQUIRED, false, null)); HierarchicalTypeDefinition columnClsDef = TypesUtil - .createClassTypeDef(COLUMN_TYPE, null, attrDef("name", DataTypes.STRING_TYPE), - attrDef("dataType", DataTypes.STRING_TYPE), attrDef("comment", DataTypes.STRING_TYPE)); + .createClassTypeDef(COLUMN_TYPE, null, attrDef("name", DataTypes.STRING_TYPE), + attrDef("dataType", DataTypes.STRING_TYPE), attrDef("comment", DataTypes.STRING_TYPE)); HierarchicalTypeDefinition tblClsDef = TypesUtil - .createClassTypeDef(HIVE_TABLE_TYPE, ImmutableList.of("DataSet"), - attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE), - attrDef("lastAccessTime", DataTypes.LONG_TYPE), attrDef("tableType", DataTypes.STRING_TYPE), - attrDef("temporary", DataTypes.BOOLEAN_TYPE), - new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.REQUIRED, false, null), - // todo - uncomment this, something is broken - // new AttributeDefinition("sd", STORAGE_DESC_TYPE, - // Multiplicity.REQUIRED, true, null), - new AttributeDefinition("columns", DataTypes.arrayTypeName(COLUMN_TYPE), - Multiplicity.COLLECTION, true, null)); + .createClassTypeDef(HIVE_TABLE_TYPE, ImmutableList.of("DataSet"), + attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE), + attrDef("lastAccessTime", DataTypes.LONG_TYPE), attrDef("tableType", DataTypes.STRING_TYPE), + attrDef("temporary", DataTypes.BOOLEAN_TYPE), + new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.REQUIRED, false, null), + // todo - uncomment this, something is broken + // new AttributeDefinition("sd", STORAGE_DESC_TYPE, + // Multiplicity.REQUIRED, true, null), + new AttributeDefinition("columns", DataTypes.arrayTypeName(COLUMN_TYPE), + Multiplicity.COLLECTION, true, null)); HierarchicalTypeDefinition loadProcessClsDef = TypesUtil - .createClassTypeDef(HIVE_PROCESS_TYPE, ImmutableList.of("Process"), - attrDef("userName", DataTypes.STRING_TYPE), attrDef("startTime", DataTypes.LONG_TYPE), - attrDef("endTime", DataTypes.LONG_TYPE), - attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), - attrDef("queryPlan", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), - attrDef("queryId", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), - attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED)); + .createClassTypeDef(HIVE_PROCESS_TYPE, ImmutableList.of("Process"), + attrDef("userName", DataTypes.STRING_TYPE), attrDef("startTime", DataTypes.LONG_TYPE), + attrDef("endTime", DataTypes.LONG_TYPE), + attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), + attrDef("queryPlan", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), + attrDef("queryId", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), + attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED)); HierarchicalTypeDefinition viewClsDef = TypesUtil - .createClassTypeDef(VIEW_TYPE, null, attrDef("name", DataTypes.STRING_TYPE), - new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.REQUIRED, false, null), - new AttributeDefinition("inputTables", DataTypes.arrayTypeName(HIVE_TABLE_TYPE), - Multiplicity.COLLECTION, false, null)); + .createClassTypeDef(VIEW_TYPE, null, attrDef("name", DataTypes.STRING_TYPE), + new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.REQUIRED, false, null), + new AttributeDefinition("inputTables", DataTypes.arrayTypeName(HIVE_TABLE_TYPE), + Multiplicity.COLLECTION, false, null)); HierarchicalTypeDefinition dimTraitDef = TypesUtil.createTraitTypeDef("Dimension", null); @@ -349,8 +368,8 @@ public class HiveLineageServiceTest { HierarchicalTypeDefinition jdbcTraitDef = TypesUtil.createTraitTypeDef("JdbcAccess", null); return TypeUtils.getTypesDef(ImmutableList.of(), ImmutableList.of(), - ImmutableList.of(dimTraitDef, factTraitDef, piiTraitDef, metricTraitDef, etlTraitDef, jdbcTraitDef), - ImmutableList.of(dbClsDef, storageDescClsDef, columnClsDef, tblClsDef, loadProcessClsDef, viewClsDef)); + ImmutableList.of(dimTraitDef, factTraitDef, piiTraitDef, metricTraitDef, etlTraitDef, jdbcTraitDef), + ImmutableList.of(dbClsDef, storageDescClsDef, columnClsDef, tblClsDef, loadProcessClsDef, viewClsDef)); } AttributeDefinition attrDef(String name, IDataType dT) { @@ -362,7 +381,7 @@ public class HiveLineageServiceTest { } AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m, boolean isComposite, - String reverseAttributeName) { + String reverseAttributeName) { Preconditions.checkNotNull(name); Preconditions.checkNotNull(dT); return new AttributeDefinition(name, dT.getName(), m, isComposite, reverseAttributeName); @@ -372,62 +391,62 @@ public class HiveLineageServiceTest { Id salesDB = database("Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales"); Referenceable sd = - storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true); + storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true); List salesFactColumns = ImmutableList - .of(column("time_id", "int", "time id"), column("product_id", "int", "product id"), - column("customer_id", "int", "customer id", "PII"), - column("sales", "double", "product id", "Metric")); + .of(column("time_id", "int", "time id"), column("product_id", "int", "product id"), + column("customer_id", "int", "customer id", "PII"), + column("sales", "double", "product id", "Metric")); Id salesFact = table("sales_fact", "sales fact table", salesDB, sd, "Joe", "Managed", salesFactColumns, "Fact"); List timeDimColumns = ImmutableList - .of(column("time_id", "int", "time id"), column("dayOfYear", "int", "day Of Year"), - column("weekDay", "int", "week Day")); + .of(column("time_id", "int", "time id"), column("dayOfYear", "int", "day Of Year"), + column("weekDay", "int", "week Day")); Id timeDim = table("time_dim", "time dimension table", salesDB, sd, "John Doe", "External", timeDimColumns, - "Dimension"); + "Dimension"); Id reportingDB = - database("Reporting", "reporting database", "Jane BI", "hdfs://host:8000/apps/warehouse/reporting"); + database("Reporting", "reporting database", "Jane BI", "hdfs://host:8000/apps/warehouse/reporting"); Id salesFactDaily = - table("sales_fact_daily_mv", "sales fact daily materialized view", reportingDB, sd, "Joe BI", "Managed", - salesFactColumns, "Metric"); + table("sales_fact_daily_mv", "sales fact daily materialized view", reportingDB, sd, "Joe BI", "Managed", + salesFactColumns, "Metric"); loadProcess("loadSalesDaily", "hive query for daily summary", "John ETL", ImmutableList.of(salesFact, timeDim), - ImmutableList.of(salesFactDaily), "create table as select ", "plan", "id", "graph", "ETL"); + ImmutableList.of(salesFactDaily), "create table as select ", "plan", "id", "graph", "ETL"); List productDimColumns = ImmutableList - .of(column("product_id", "int", "product id"), column("product_name", "string", "product name"), - column("brand_name", "int", "brand name")); + .of(column("product_id", "int", "product id"), column("product_name", "string", "product name"), + column("brand_name", "int", "brand name")); Id productDim = - table("product_dim", "product dimension table", salesDB, sd, "John Doe", "Managed", productDimColumns, - "Dimension"); + table("product_dim", "product dimension table", salesDB, sd, "John Doe", "Managed", productDimColumns, + "Dimension"); view("product_dim_view", reportingDB, ImmutableList.of(productDim), "Dimension", "JdbcAccess"); List customerDimColumns = ImmutableList.of(column("customer_id", "int", "customer id", "PII"), - column("name", "string", "customer name", "PII"), - column("address", "string", "customer address", "PII")); + column("name", "string", "customer name", "PII"), + column("address", "string", "customer address", "PII")); Id customerDim = - table("customer_dim", "customer dimension table", salesDB, sd, "fetl", "External", customerDimColumns, - "Dimension"); + table("customer_dim", "customer dimension table", salesDB, sd, "fetl", "External", customerDimColumns, + "Dimension"); view("customer_dim_view", reportingDB, ImmutableList.of(customerDim), "Dimension", "JdbcAccess"); Id salesFactMonthly = - table("sales_fact_monthly_mv", "sales fact monthly materialized view", reportingDB, sd, "Jane BI", - "Managed", salesFactColumns, "Metric"); + table("sales_fact_monthly_mv", "sales fact monthly materialized view", reportingDB, sd, "Jane BI", + "Managed", salesFactColumns, "Metric"); loadProcess("loadSalesMonthly", "hive query for monthly summary", "John ETL", ImmutableList.of(salesFactDaily), - ImmutableList.of(salesFactMonthly), "create table as select ", "plan", "id", "graph", "ETL"); + ImmutableList.of(salesFactMonthly), "create table as select ", "plan", "id", "graph", "ETL"); } Id database(String name, String description, String owner, String locationUri, String... traitNames) - throws Exception { + throws Exception { Referenceable referenceable = new Referenceable(DATABASE_TYPE, traitNames); referenceable.set("name", name); referenceable.set("description", description); @@ -439,7 +458,7 @@ public class HiveLineageServiceTest { } Referenceable storageDescriptor(String location, String inputFormat, String outputFormat, boolean compressed) - throws Exception { + throws Exception { Referenceable referenceable = new Referenceable(STORAGE_DESC_TYPE); referenceable.set("location", location); referenceable.set("inputFormat", inputFormat); @@ -459,7 +478,7 @@ public class HiveLineageServiceTest { } Id table(String name, String description, Id dbId, Referenceable sd, String owner, String tableType, - List columns, String... traitNames) throws Exception { + List columns, String... traitNames) throws Exception { Referenceable referenceable = new Referenceable(HIVE_TABLE_TYPE, traitNames); referenceable.set("name", name); referenceable.set("description", description); @@ -479,8 +498,8 @@ public class HiveLineageServiceTest { } Id loadProcess(String name, String description, String user, List inputTables, List outputTables, - String queryText, String queryPlan, String queryId, String queryGraph, String... traitNames) - throws Exception { + String queryText, String queryPlan, String queryId, String queryGraph, String... traitNames) + throws Exception { Referenceable referenceable = new Referenceable(HIVE_PROCESS_TYPE, traitNames); referenceable.set("name", name); referenceable.set("description", description); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java index fb697d0..487ac84 100755 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java @@ -20,9 +20,11 @@ package org.apache.atlas.repository.graph; import com.google.common.collect.ImmutableList; import com.thinkaurelius.titan.core.TitanGraph; +import com.thinkaurelius.titan.core.util.TitanCleanup; import com.tinkerpop.blueprints.Compare; import com.tinkerpop.blueprints.GraphQuery; import com.tinkerpop.blueprints.Vertex; +import org.apache.atlas.GraphTransaction; import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.TestUtils; import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; @@ -51,6 +53,7 @@ import org.apache.commons.lang.RandomStringUtils; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONObject; import org.testng.Assert; +import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Guice; import org.testng.annotations.Test; @@ -105,12 +108,23 @@ public class GraphBackedMetadataRepositoryTest { createHiveTypes(); } -/* - @AfterMethod + + @AfterClass public void tearDown() throws Exception { - TestUtils.dumpGraph(graphProvider.get()); + TypeSystem.getInstance().reset(); + try { + //TODO - Fix failure during shutdown while using BDB + graphProvider.get().shutdown(); + } catch(Exception e) { + e.printStackTrace(); + } + try { + TitanCleanup.clear(graphProvider.get()); + } catch(Exception e) { + e.printStackTrace(); + } } -*/ + @Test public void testSubmitEntity() throws Exception { @@ -189,7 +203,8 @@ public class GraphBackedMetadataRepositoryTest { System.out.println("*** table = " + table); } - private String getGUID() { + @GraphTransaction + String getGUID() { Vertex tableVertex = getTableEntityVertex(); String guid = tableVertex.getProperty(Constants.GUID_PROPERTY_KEY); @@ -199,7 +214,8 @@ public class GraphBackedMetadataRepositoryTest { return guid; } - private Vertex getTableEntityVertex() { + @GraphTransaction + Vertex getTableEntityVertex() { TitanGraph graph = graphProvider.get(); GraphQuery query = graph.query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, Compare.EQUAL, TABLE_TYPE); Iterator results = query.vertices().iterator(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java index c774123..6a8129f 100755 --- a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java @@ -19,6 +19,7 @@ package org.apache.atlas.repository.typestore; import com.thinkaurelius.titan.core.TitanGraph; +import com.thinkaurelius.titan.core.util.TitanCleanup; import com.tinkerpop.blueprints.Direction; import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Vertex; @@ -39,6 +40,7 @@ import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; import org.apache.atlas.typesystem.types.StructTypeDefinition; import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.TypeSystem; +import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Guice; import org.testng.annotations.Test; @@ -63,6 +65,17 @@ public class GraphBackedTypeStoreTest { TestUtils.defineDeptEmployeeTypes(ts); } + @AfterClass + public void tearDown() throws Exception { + ts.reset(); + graphProvider.get().shutdown(); + try { + TitanCleanup.clear(graphProvider.get()); + } catch(Exception e) { + e.printStackTrace(); + } + } + @Test @GraphTransaction public void testStore() throws AtlasException { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/repository/src/test/resources/application.properties ---------------------------------------------------------------------- diff --git a/repository/src/test/resources/application.properties b/repository/src/test/resources/application.properties index 20c0c51..d0eaa8c 100755 --- a/repository/src/test/resources/application.properties +++ b/repository/src/test/resources/application.properties @@ -19,15 +19,35 @@ ######### Graph Database Configs ######### #Refer http://s3.thinkaurelius.com/docs/titan/0.5.1/titan-config-ref.html # Graph Storage -atlas.graph.storage.backend=inmemory +atlas.graph.storage.backend=${titan.storage.backend} -# Graph Search Index -atlas.graph.index.search.backend=elasticsearch +#Berkeley storage directory +atlas.graph.storage.directory=target/data/berkley + +#hbase +#For standalone mode , specify localhost +#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2 +atlas.graph.storage.hostname=${titan.storage.hostname} + +# Graph Search Index Backend +atlas.graph.index.search.backend=${titan.index.backend} + +#lucene +#atlas.graph.index.search.directory=target/data/lucene + +#elasticsearch atlas.graph.index.search.directory=./target/data/es atlas.graph.index.search.elasticsearch.client-only=false atlas.graph.index.search.elasticsearch.local-mode=true +atlas.graph.index.search.elasticsearch.create.sleep=2000 + +#solr in cloud mode +atlas.graph.index.search.solr.mode=cloud +atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address} +#solr in http mode +atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr ######### Hive Lineage Configs ######### #atlas.lineage.hive.table.type.name=DataSet http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/src/conf/application.properties ---------------------------------------------------------------------- diff --git a/src/conf/application.properties b/src/conf/application.properties index c6b16cc..5c8c59a 100755 --- a/src/conf/application.properties +++ b/src/conf/application.properties @@ -21,6 +21,23 @@ atlas.graph.storage.backend=berkeleyje atlas.graph.storage.directory=data/berkley +#Hbase as stoarge backend +#hbase +#For standalone mode , specify localhost +#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2 +#atlas.graph.storage.hostname=localhost + +#Solr +#atlas.graph.index.search.backend=solr + +# Solr cloud mode properties +#atlas.graph.index.search.solr.mode=cloud +#atlas.graph.index.search.solr.zookeeper-url=localhost:2181 + +#Solr http mode properties +#atlas.graph.index.search.solr.mode=http +#atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr + # Graph Search Index atlas.graph.index.search.backend=elasticsearch atlas.graph.index.search.directory=data/es http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/pom.xml ---------------------------------------------------------------------- diff --git a/webapp/pom.xml b/webapp/pom.xml index 0ae4a05..86e1ebf 100755 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -116,13 +116,13 @@ - org.mortbay.jetty - jetty + org.eclipse.jetty + jetty-server - org.mortbay.jetty - jetty-plus + org.eclipse.jetty + jetty-webapp @@ -171,8 +171,13 @@ - org.mortbay.jetty - jsp-2.1 + org.eclipse.jetty + jetty-jsp + + + + org.codehaus.jackson + jackson-core-asl @@ -192,36 +197,36 @@ ../dashboard/v2/ - - install node and npm - - install-node-and-npm - - - v0.10.30 - 1.4.3 - - - - - npm install - - npm - - - install - - - - - grunt dist - - grunt - - - build - - + + install node and npm + + install-node-and-npm + + + v0.10.30 + 1.4.3 + + + + + npm install + + npm + + + install + + + + + grunt dist + + grunt + + + build + + @@ -245,7 +250,17 @@ - + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + org.apache.maven.plugins maven-war-plugin @@ -260,7 +275,16 @@ src/main/webapp/WEB-INF WEB-INF + + ${project.build.directory}/test-classes + WEB-INF/classes + + + + true + + @@ -312,30 +336,24 @@ - org.mortbay.jetty - maven-jetty-plugin - ${jetty.version} + org.eclipse.jetty + jetty-maven-plugin ${skipTests} - - - - 21000 - 60000 - - - ${project.build.directory}/atlas-webapp-${project.version} - / - true + + 21000 + 60000 + + ${project.build.directory}/atlas-webapp-${project.version}.war + true + + + / + webapp/src/test/webapp/WEB-INF/web.xml + + + true atlas.log.dir @@ -364,11 +382,8 @@ start-jetty pre-integration-test - run + deploy-war - - true - stop-jetty http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java b/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java index 18ce17c..b643f13 100755 --- a/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java +++ b/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java @@ -20,14 +20,20 @@ package org.apache.atlas.web.listeners; import com.google.inject.Guice; import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.Provider; +import com.google.inject.TypeLiteral; import com.google.inject.servlet.GuiceServletContextListener; import com.sun.jersey.api.core.PackagesResourceConfig; import com.sun.jersey.guice.JerseyServletModule; import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.thinkaurelius.titan.core.TitanGraph; +import com.tinkerpop.blueprints.Graph; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; import org.apache.atlas.PropertiesUtil; import org.apache.atlas.RepositoryMetadataModule; +import org.apache.atlas.repository.graph.GraphProvider; import org.apache.atlas.web.filters.AtlasAuthenticationFilter; import org.apache.atlas.web.filters.AuditFilter; import org.apache.commons.configuration.ConfigurationException; @@ -35,6 +41,7 @@ import org.apache.commons.configuration.PropertiesConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.inject.Inject; import javax.servlet.ServletContextEvent; import java.util.HashMap; import java.util.Map; @@ -45,7 +52,7 @@ public class GuiceServletConfig extends GuiceServletContextListener { private static final String GUICE_CTX_PARAM = "guice.packages"; static final String HTTP_AUTHENTICATION_ENABLED = "atlas.http.authentication.enabled"; - private Injector injector; + protected Injector injector; @Override protected Injector getInjector() { @@ -106,5 +113,11 @@ public class GuiceServletConfig extends GuiceServletContextListener { @Override public void contextDestroyed(ServletContextEvent servletContextEvent) { super.contextDestroyed(servletContextEvent); + if(injector != null) { + TypeLiteral> graphProviderType = new TypeLiteral>() {}; + Provider> graphProvider = injector.getProvider(Key.get(graphProviderType)); + final Graph graph = graphProvider.get().get(); + graph.shutdown(); + } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java b/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java index 88200f0..16591d8 100755 --- a/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java +++ b/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java @@ -20,10 +20,13 @@ package org.apache.atlas.web.service; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; -import org.mortbay.jetty.Connector; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.webapp.WebAppContext; +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.webapp.WebAppClassLoader; +import org.eclipse.jetty.webapp.WebAppContext; import java.io.IOException; @@ -40,6 +43,7 @@ public class EmbeddedServer { server.addConnector(connector); WebAppContext application = new WebAppContext(path, "/"); + application.setClassLoader(Thread.currentThread().getContextClassLoader()); server.setHandler(application); } @@ -52,19 +56,21 @@ public class EmbeddedServer { } protected Connector getConnector(int port) throws IOException { - Connector connector = new SocketConnector(); - connector.setPort(port); - connector.setHost("0.0.0.0"); + HttpConfiguration http_config = new HttpConfiguration(); // this is to enable large header sizes when Kerberos is enabled with AD - final Integer bufferSize = getBufferSize(); - connector.setHeaderBufferSize(bufferSize); - connector.setRequestBufferSize(bufferSize); + final int bufferSize = getBufferSize(); + http_config.setResponseHeaderSize(bufferSize); + http_config.setRequestHeaderSize(bufferSize); + ServerConnector connector = new ServerConnector(server, new HttpConnectionFactory(http_config)); + connector.setPort(port); + connector.setHost("0.0.0.0"); + server.addConnector(connector); return connector; } - private Integer getBufferSize() { + protected Integer getBufferSize() { try { PropertiesConfiguration configuration = new PropertiesConfiguration("application.properties"); return configuration.getInt("atlas.jetty.request.buffer.size", DEFAULT_BUFFER_SIZE);