Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 2BBAF200C2A for ; Wed, 1 Mar 2017 22:27:04 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 2A207160B56; Wed, 1 Mar 2017 21:27:04 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id F0FDF160B70 for ; Wed, 1 Mar 2017 22:27:01 +0100 (CET) Received: (qmail 25476 invoked by uid 500); 1 Mar 2017 21:27:01 -0000 Mailing-List: contact commits-help@atlas.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@atlas.incubator.apache.org Delivered-To: mailing list commits@atlas.incubator.apache.org Received: (qmail 25467 invoked by uid 99); 1 Mar 2017 21:27:01 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd3-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 01 Mar 2017 21:27:01 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd3-us-west.apache.org (ASF Mail Server at spamd3-us-west.apache.org) with ESMTP id 9308C1895B9 for ; Wed, 1 Mar 2017 21:27:00 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd3-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -6.567 X-Spam-Level: X-Spam-Status: No, score=-6.567 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-2.999, SPF_NEUTRAL=0.652] autolearn=disabled Received: from mx1-lw-eu.apache.org ([10.40.0.8]) by localhost (spamd3-us-west.apache.org [10.40.0.10]) (amavisd-new, port 10024) with ESMTP id 04yTGHV9YjuJ for ; Wed, 1 Mar 2017 21:26:49 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-eu.apache.org (ASF Mail Server at mx1-lw-eu.apache.org) with SMTP id D82015FDDC for ; Wed, 1 Mar 2017 21:26:46 +0000 (UTC) Received: (qmail 24719 invoked by uid 99); 1 Mar 2017 21:26:46 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 01 Mar 2017 21:26:46 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D93AFE00B7; Wed, 1 Mar 2017 21:26:45 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jnhagelberg@apache.org To: commits@atlas.incubator.apache.org Date: Wed, 01 Mar 2017 21:26:47 -0000 Message-Id: <1df9b1deef5146b8b885285458f04b20@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [3/3] incubator-atlas git commit: ATLAS-695: Add Titan 1 project to Atlas archived-at: Wed, 01 Mar 2017 21:27:04 -0000 ATLAS-695: Add Titan 1 project to Atlas Signed-off-by: Jeff Hagelberg Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/6fd04d9a Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/6fd04d9a Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/6fd04d9a Branch: refs/heads/master Commit: 6fd04d9a8c56610a855b3950040ace9b6699c38b Parents: 46b1b36 Author: Neeru Gupta Authored: Wed Mar 1 15:03:32 2017 -0500 Committer: Jeff Hagelberg Committed: Wed Mar 1 16:13:54 2017 -0500 ---------------------------------------------------------------------- catalog/pom.xml | 79 ++- .../apache/atlas/catalog/query/BaseQuery.java | 9 +- .../catalog/query/AtlasEntityQueryTest.java | 14 +- .../utils/IteratorToIterableAdapter.java | 38 ++ graphdb/graphdb-impls/pom.xml | 17 +- graphdb/pom.xml | 4 +- graphdb/titan0/pom.xml | 27 +- .../repository/graphdb/titan0/Titan0Graph.java | 2 +- .../atlas/utils/IteratorToIterableAdapter.java | 38 -- graphdb/titan1/pom.xml | 333 ++++++++++++ graphdb/titan1/readme.txt | 87 ++++ .../graphdb/titan1/GraphDbObjectFactory.java | 118 +++++ .../repository/graphdb/titan1/Titan1Edge.java | 54 ++ .../graphdb/titan1/Titan1Element.java | 258 ++++++++++ .../repository/graphdb/titan1/Titan1Graph.java | 405 +++++++++++++++ .../graphdb/titan1/Titan1GraphDatabase.java | 176 +++++++ .../graphdb/titan1/Titan1GraphIndex.java | 100 ++++ .../graphdb/titan1/Titan1GraphManagement.java | 177 +++++++ .../graphdb/titan1/Titan1IndexQuery.java | 79 +++ .../graphdb/titan1/Titan1PropertyKey.java | 77 +++ .../repository/graphdb/titan1/Titan1Vertex.java | 102 ++++ .../graphdb/titan1/Titan1VertexQuery.java | 66 +++ .../graphdb/titan1/TitanObjectFactory.java | 85 +++ .../graphson/AtlasElementPropertyConfig.java | 136 +++++ .../titan1/graphson/AtlasGraphSONMode.java | 43 ++ .../titan1/graphson/AtlasGraphSONTokens.java | 51 ++ .../titan1/graphson/AtlasGraphSONUtility.java | 513 +++++++++++++++++++ .../titan1/query/NativeTitan1GraphQuery.java | 85 +++ .../graphdb/titan1/query/Titan1GraphQuery.java | 55 ++ .../titan1/serializer/BigDecimalSerializer.java | 48 ++ .../titan1/serializer/BigIntegerSerializer.java | 46 ++ .../titan1/serializer/StringListSerializer.java | 53 ++ .../serializer/TypeCategorySerializer.java | 31 ++ .../services/javax.script.ScriptEngineFactory | 1 + .../titan1/AbstractGraphDatabaseTest.java | 191 +++++++ .../graphdb/titan1/GraphQueryTest.java | 451 ++++++++++++++++ .../graphdb/titan1/Titan1DatabaseTest.java | 432 ++++++++++++++++ .../graphdb/titan1/TitanGraphProviderTest.java | 77 +++ .../test/resources/atlas-application.properties | 97 ++++ pom.xml | 35 +- release-log.txt | 1 + .../test/java/org/apache/atlas/TestUtils.java | 10 + .../GraphBackedDiscoveryServiceTest.java | 15 +- .../atlas/lineage/EntityLineageServiceTest.java | 31 +- .../audit/AuditRepositoryTestBase.java | 4 + .../audit/HBaseBasedAuditRepositoryTest.java | 5 + webapp/pom.xml | 170 +++--- 47 files changed, 4748 insertions(+), 178 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/catalog/pom.xml ---------------------------------------------------------------------- diff --git a/catalog/pom.xml b/catalog/pom.xml index 00e56dc..55b8dff 100755 --- a/catalog/pom.xml +++ b/catalog/pom.xml @@ -24,29 +24,34 @@ org.apache.atlas apache-atlas - 0.8-incubating-SNAPSHOT + 0.8-incubating-SNAPSHOT atlas-catalog Apache Atlas Business Catalog Module Apache Atlas Business Catalog jar - 2.6.0 0.5.4 + 4.8.1 - - - - org.apache.atlas - atlas-repository - + + + titan1 + + false + + + 4.10.4 + + + + + org.apache.atlas atlas-repository - tests - test @@ -64,11 +69,43 @@ atlas-graphdb-api - - - org.apache.atlas - atlas-graphdb-titan0 - ${project.version} + + + com.tinkerpop.blueprints + blueprints-core + ${tinkerpop.version} + + + com.tinkerpop.gremlin + gremlin-java + ${tinkerpop.version} + + + + com.thinkaurelius.titan + titan-core + provided + + + + org.apache.lucene + lucene-core + ${lucene.version} + + + + org.apache.lucene + lucene-queryparser + ${lucene.version} + + + + org.apache.lucene + lucene-analyzers-common + ${lucene.version} @@ -96,18 +133,6 @@ json-simple - - com.tinkerpop.blueprints - blueprints-core - ${tinkerpop.version} - - - - com.tinkerpop.gremlin - gremlin-java - ${tinkerpop.version} - - org.easymock http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/catalog/src/main/java/org/apache/atlas/catalog/query/BaseQuery.java ---------------------------------------------------------------------- diff --git a/catalog/src/main/java/org/apache/atlas/catalog/query/BaseQuery.java b/catalog/src/main/java/org/apache/atlas/catalog/query/BaseQuery.java index 6a2ba53..18d87ea 100644 --- a/catalog/src/main/java/org/apache/atlas/catalog/query/BaseQuery.java +++ b/catalog/src/main/java/org/apache/atlas/catalog/query/BaseQuery.java @@ -30,10 +30,10 @@ import org.apache.atlas.catalog.exception.ResourceNotFoundException; import org.apache.atlas.catalog.projection.Projection; import org.apache.atlas.catalog.projection.ProjectionResult; import org.apache.atlas.repository.Constants; -import org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase; +import org.apache.atlas.repository.graph.AtlasGraphProvider; +import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.typesystem.persistence.Id; -import com.thinkaurelius.titan.core.TitanGraph; import com.tinkerpop.blueprints.Compare; import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.gremlin.java.GremlinPipeline; @@ -165,10 +165,9 @@ public abstract class BaseQuery implements AtlasQuery { return request; } - //todo: abstract // Underlying method is synchronized and caches the graph in a static field - protected TitanGraph getGraph() { - return Titan0GraphDatabase.getGraphInstance(); + protected AtlasGraph getGraph() { + return AtlasGraphProvider.getGraphInstance(); } protected VertexWrapper wrapVertex(Vertex v) { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/catalog/src/test/java/org/apache/atlas/catalog/query/AtlasEntityQueryTest.java ---------------------------------------------------------------------- diff --git a/catalog/src/test/java/org/apache/atlas/catalog/query/AtlasEntityQueryTest.java b/catalog/src/test/java/org/apache/atlas/catalog/query/AtlasEntityQueryTest.java index 901a549..528c83a 100644 --- a/catalog/src/test/java/org/apache/atlas/catalog/query/AtlasEntityQueryTest.java +++ b/catalog/src/test/java/org/apache/atlas/catalog/query/AtlasEntityQueryTest.java @@ -22,10 +22,12 @@ import com.thinkaurelius.titan.core.TitanGraph; import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.gremlin.java.GremlinPipeline; import com.tinkerpop.pipes.Pipe; + import org.apache.atlas.catalog.Request; import org.apache.atlas.catalog.VertexWrapper; import org.apache.atlas.catalog.definition.ResourceDefinition; import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graphdb.AtlasGraph; import org.easymock.Capture; import org.testng.annotations.Test; @@ -44,7 +46,7 @@ public class AtlasEntityQueryTest { //todo: add tests for instance query and getInitialPipeline() @Test public void testExecute_Collection() throws Exception { - TitanGraph graph = createStrictMock(TitanGraph.class); + AtlasGraph graph = createStrictMock(AtlasGraph.class); QueryExpression expression = createStrictMock(QueryExpression.class); ResourceDefinition resourceDefinition = createStrictMock(ResourceDefinition.class); Request request = createStrictMock(Request.class); @@ -107,7 +109,7 @@ public class AtlasEntityQueryTest { @Test public void testExecute_Collection_rollbackOnException() throws Exception { - TitanGraph graph = createStrictMock(TitanGraph.class); + AtlasGraph graph = createStrictMock(AtlasGraph.class); QueryExpression expression = createStrictMock(QueryExpression.class); ResourceDefinition resourceDefinition = createStrictMock(ResourceDefinition.class); Request request = createStrictMock(Request.class); @@ -151,7 +153,7 @@ public class AtlasEntityQueryTest { @Test public void testExecute_Collection_update() throws Exception { - TitanGraph graph = createStrictMock(TitanGraph.class); + AtlasGraph graph = createStrictMock(AtlasGraph.class); QueryExpression expression = createStrictMock(QueryExpression.class); ResourceDefinition resourceDefinition = createStrictMock(ResourceDefinition.class); Request request = createStrictMock(Request.class); @@ -227,7 +229,7 @@ public class AtlasEntityQueryTest { private final GremlinPipeline initialPipeline; private final Pipe queryPipe; private final Pipe notDeletedPipe; - private final TitanGraph graph; + private final AtlasGraph graph; private final VertexWrapper vWrapper; public TestAtlasEntityQuery(QueryExpression queryExpression, @@ -236,7 +238,7 @@ public class AtlasEntityQueryTest { GremlinPipeline initialPipeline, Pipe queryPipe, Pipe notDeletedPipe, - TitanGraph graph, + AtlasGraph graph, VertexWrapper vWrapper) { super(queryExpression, resourceDefinition, request); @@ -263,7 +265,7 @@ public class AtlasEntityQueryTest { } @Override - protected TitanGraph getGraph() { + protected AtlasGraph getGraph() { return graph; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/utils/IteratorToIterableAdapter.java ---------------------------------------------------------------------- diff --git a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/utils/IteratorToIterableAdapter.java b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/utils/IteratorToIterableAdapter.java new file mode 100644 index 0000000..6d3c51e --- /dev/null +++ b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/utils/IteratorToIterableAdapter.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.graphdb.utils; + +import java.util.Iterator; + +/** + * Adapter class that allows an Iterator to be presented as an instance of java.util.Iterable. + */ +public final class IteratorToIterableAdapter implements Iterable { + + private final Iterator wrapped; + + public IteratorToIterableAdapter(Iterator wrapped) { + this.wrapped = wrapped; + } + + @Override + public Iterator iterator() { + return wrapped; + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/graphdb/graphdb-impls/pom.xml ---------------------------------------------------------------------- diff --git a/graphdb/graphdb-impls/pom.xml b/graphdb/graphdb-impls/pom.xml index 57b5137..4a1857f 100644 --- a/graphdb/graphdb-impls/pom.xml +++ b/graphdb/graphdb-impls/pom.xml @@ -33,25 +33,20 @@ should be configured to exclude all but the proper dependency --> - + Apache Atlas Graph Database Implementation Dependencies Apache Atlas Graph Database Implementation Dependencies pom - - + + org.apache.atlas + atlas-graphdb-titan1 + ${project.version} + org.apache.atlas atlas-graphdb-titan0 ${project.version} - - - - http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/graphdb/pom.xml ---------------------------------------------------------------------- diff --git a/graphdb/pom.xml b/graphdb/pom.xml index 2d303e4..424acc0 100644 --- a/graphdb/pom.xml +++ b/graphdb/pom.xml @@ -19,6 +19,7 @@ 4.0.0 + apache-atlas org.apache.atlas @@ -32,8 +33,9 @@ api - titan0 common graphdb-impls + titan0 + titan1 http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/graphdb/titan0/pom.xml ---------------------------------------------------------------------- diff --git a/graphdb/titan0/pom.xml b/graphdb/titan0/pom.xml index 9d88a72..060fda9 100644 --- a/graphdb/titan0/pom.xml +++ b/graphdb/titan0/pom.xml @@ -34,9 +34,31 @@ 2.6.0 0.5.4 - 14.0 + false + 14.0 + + + titan0 + + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.18.1 + + false + + + + + + + @@ -150,7 +172,7 @@ org.apache.maven.plugins @@ -188,7 +210,6 @@ atlas.shaded.titan.guava - true http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java ---------------------------------------------------------------------- diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java index 75ea545..be38d5b 100644 --- a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java +++ b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java @@ -45,8 +45,8 @@ import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.graphdb.GremlinVersion; import org.apache.atlas.repository.graphdb.titan0.query.Titan0GraphQuery; +import org.apache.atlas.repository.graphdb.utils.IteratorToIterableAdapter; import org.apache.atlas.typesystem.types.IDataType; -import org.apache.atlas.utils.IteratorToIterableAdapter; import com.google.common.base.Function; import com.google.common.collect.Iterables; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/graphdb/titan0/src/main/java/org/apache/atlas/utils/IteratorToIterableAdapter.java ---------------------------------------------------------------------- diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/utils/IteratorToIterableAdapter.java b/graphdb/titan0/src/main/java/org/apache/atlas/utils/IteratorToIterableAdapter.java deleted file mode 100644 index ed7c2a7..0000000 --- a/graphdb/titan0/src/main/java/org/apache/atlas/utils/IteratorToIterableAdapter.java +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.utils; - -import java.util.Iterator; - -/** - * Adapter class that allows an Iterator to be presented as an instance of java.util.Iterable. - */ -public final class IteratorToIterableAdapter implements Iterable { - - private final Iterator wrapped; - - public IteratorToIterableAdapter(Iterator wrapped) { - this.wrapped = wrapped; - } - - @Override - public Iterator iterator() { - return wrapped; - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/graphdb/titan1/pom.xml ---------------------------------------------------------------------- diff --git a/graphdb/titan1/pom.xml b/graphdb/titan1/pom.xml new file mode 100644 index 0000000..ad289e5 --- /dev/null +++ b/graphdb/titan1/pom.xml @@ -0,0 +1,333 @@ + + + + + 4.0.0 + + atlas-graphdb + org.apache.atlas + 0.8-incubating-SNAPSHOT + + atlas-graphdb-titan1 + Apache Atlas Titan 1.0.0 Graph DB Impl + Apache Atlas Titan 1.0.0 GraphDB Impl + jar + + + 3.0.1-incubating + 1.0.0 + false + + + + titan1 + + 1.8 + + + + + org.apache.maven.plugins + maven-enforcer-plugin + 1.4.1 + + + enforce-java-8 + + enforce + + + + + 1.8.0 + + + 3.1.0 + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.18.1 + + false + + + + maven-compiler-plugin + 3.1 + + false + false + + org.apache.atlas + + + + + + + org.apache.atlas.repository.graphdb.titan1.Titan1GraphDatabase + + + + + + org.apache.atlas + atlas-graphdb-common + ${project.version} + + + org.apache.atlas + atlas-graphdb-api + + provided + + + + com.google.inject + guice + provided + + + + commons-configuration + commons-configuration + provided + + + + + com.thinkaurelius.titan + titan-hbase + ${titan.version} + + + + com.thinkaurelius.titan + titan-core + ${titan.version} + + + + com.thinkaurelius.titan + titan-solr + ${titan.version} + + + + javax.servlet + servlet-api + + + + + + org.apache.hbase + hbase-client + + + + com.vividsolutions + jts + + + + org.apache.solr + solr-core + + + + org.apache.solr + solr-solrj + + + + com.thinkaurelius.titan + titan-es + ${titan.version} + + + + com.thinkaurelius.titan + titan-berkeleyje + ${titan.version} + + + + com.thinkaurelius.titan + titan-lucene + ${titan.version} + + + + org.testng + testng + + + + org.mockito + mockito-all + + + + + com.google.guava + guava + 18.0 + + + + + org.apache.tinkerpop + gremlin-core + 3.0.1-incubating + + + + org.apache.tinkerpop + tinkergraph-gremlin + 3.0.1-incubating + + + + + + + + maven-compiler-plugin + 3.1 + + true + true + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.18.1 + + true + true + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.3 + + + package + + shade + + + false + + + + org.slf4j:* + + + + + com.google.common + org.apache.atlas.shaded.com.google.common + + + + + + + + + + + + + + + + + + com.thinkaurelius.titan + titan-core + 1.0.0 + + + + com.tinkerpop.rexster + rexster-core + + + com.tinkerpop.rexster + rexster-server + + + + com.tinkerpop + frames + + + com.esotericsoftware.reflectasm + reflectasm + + + org.ow2.asm + asm + + + org.acplt + oncrpc + + + + + + com.thinkaurelius.titan + titan-berkeleyje + 1.0.0 + + + + com.thinkaurelius.titan + titan-hbase + 1.0.0 + + + + com.thinkaurelius.titan + titan-solr + 1.0.0 + + + + + http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/graphdb/titan1/readme.txt ---------------------------------------------------------------------- diff --git a/graphdb/titan1/readme.txt b/graphdb/titan1/readme.txt new file mode 100644 index 0000000..4fcddba --- /dev/null +++ b/graphdb/titan1/readme.txt @@ -0,0 +1,87 @@ +================== +Titan 1.0.0 README +================== + +IMPORTANT: Titan 1 support in Atlas is currently a work in progress. + + +ARCHITECTURE NOTES +------------------ + +The build is currently set up to include only one of the graph backends in the Atlas war file. The configured graph +backend is determined by maven profile. The default profile is titan0. To build Atlas configured to run against Titan 1, +the titan1 profile must be enabled. Titan1 project has support for Gremlin3. Gremlin Translator translates the DSL to +gremlin3 compliant syntax when titan1 is configured as backend graph db. + + +REQUIREMENTS +-------------- + +Titan 1 requires Java 8 to be used both when building and running Atlas. While building on Java7, the java classes in +the Titan 1 project are not compiled. But an empty jar is produced. If java 8 is used, then titan 1 classes are compiled +but tests are only executed if titan1 profile is explicitly enabled. + + + +USING ATLAS WITH TITAN 1 +------------------------ + +1) Build Atlas with the titan1 maven profile enabled: + +mvn install -P dist -P titan1 + +Note that there are some tests in repository and webapp project which are skipped when running with the titan1 profile. +Please refer to known issues section below. + +This will build Atlas and run all of the tests against Titan 1. Only Atlas builds that were generated with the titan1 +maven profile enabled can be used to use Atlas with Titan 1. + +2) Configure the Atlas runtime to use Titan 1 by setting the atlas.graphdb.backend property in +ATLAS_HOME/conf/atlas-application.properties, as follows: + +atlas.graphdb.backend=org.apache.atlas.repository.graphdb.titan1.Titan1GraphDatabase + +3) Attempt to start the Atlas server. NOTE: As of this writing, Atlas fails to start (see issue 2 below). + + +KNOWN ISSUES +------------ + +1) EntityLineageService is hard-coded to generate Gremlin that is specific to Tinker Pop 2. It needs to be updated to +use GremlinExpressionFactory to generate Gremlin that is appropriate for the version of Titan being used. Currently +these tests are skipped when the titan1 profile is enabled. + +https://issues.apache.org/jira/browse/ATLAS-1579 + +2) Catalog project is hard-coded to generate Gremlin that is specific to Tinker Pop 2. It needs to be updated to use +GremlinExpressionFactory to generate Gremlin that is appropriate for the version of Titan being used. In addition, it +has direct dependencies on titan 0 / Tinker Pop 2 classes. + +https://issues.apache.org/jira/browse/ATLAS-1580 + +3) The Atlas war file startup is currently failing when Titan1 is being used. Due to the titan 0 dependencies in +catalog, the catalog jar is currently being excluded from webapp when titan1 is being used. However, Atlas appears to +have runtime dependencies on the stuff in Catalog. The war startup currently fails with the following exception: + +Caused by: java.lang.ClassNotFoundException: org.apache.atlas.catalog.exception.CatalogException + at org.codehaus.plexus.classworlds.strategy.SelfFirstStrategy.loadClass(SelfFirstStrategy.java:50) + at org.codehaus.plexus.classworlds.realm.ClassRealm.unsynchronizedLoadClass(ClassRealm.java:271) + +https://issues.apache.org/jira/browse/ATLAS-1581 + +4) There are some known test failures in webapp project. Those need to be addressed. There is work needed to refactor +webapp so that it can function without catalog until issue #2 above is resolved. + +https://issues.apache.org/jira/browse/ATLAS-1582 + +5) We cannot bundle both titan0 and titan1 in the Atlas war file because titan1 and titan0 require different versions of +the same libraries. We cannot have multiple versions of the same library on the classpath. Another issue there is that +Gremin queries are executed via the java services mechanism. Specifically, the titan 1 implemention bundles a file +called javax.script.ScriptingEngineFactory, which tells Java to +use org.apache.tinkerpop.gremlin.groovy.jsr223.GremlinGroovyScriptEngineFactory. We cannot have this file on the +classpath when using titan 0 since it would cause the Titan 0 implemention to try to execute queries using TP 3. There +may be ways of working around this, such as making Titan1Graph and Titan0Graph explicitly instantiate the +GremlinGroovyScriptEngineFactory. This was not investigated very much, though. If we combine that with shading +Titan0/1 and all of their dependencies, we might be able to bundle both the Titan 1 and Titan 0 implementations in the +Atlas war file and have things work correctly for both versions. Another possibility would be to use a custom +classloader to load the correct atlas-graphdb-titan0/1 jar during Atlas startup based on the configuration. http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/GraphDbObjectFactory.java ---------------------------------------------------------------------- diff --git a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/GraphDbObjectFactory.java b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/GraphDbObjectFactory.java new file mode 100644 index 0000000..c27bc35 --- /dev/null +++ b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/GraphDbObjectFactory.java @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.graphdb.titan1; + +import org.apache.atlas.repository.graphdb.AtlasCardinality; +import org.apache.atlas.repository.graphdb.AtlasGraphIndex; +import org.apache.atlas.repository.graphdb.titan1.query.Titan1GraphQuery; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Vertex; + +import com.thinkaurelius.titan.core.Cardinality; +import com.thinkaurelius.titan.core.PropertyKey; +import com.thinkaurelius.titan.core.schema.TitanGraphIndex; + + +/** + * Factory that serves up instances of graph database abstraction layer classes + * that correspond to Titan/Tinkerpop3 classes. + */ +public final class GraphDbObjectFactory { + + private GraphDbObjectFactory() { + + } + + /** + * Creates a Titan1Edge that corresponds to the given Gremlin Edge. + * + * @param graph The graph the edge should be created in + * @param source The gremlin edge + */ + public static Titan1Edge createEdge(Titan1Graph graph, Edge source) { + + if (source == null) { + return null; + } + return new Titan1Edge(graph, source); + } + + /** + * Creates a Titan1GraphQuery that corresponds to the given GraphQuery. + * + * @param graph the graph that is being quried + */ + public static Titan1GraphQuery createQuery(Titan1Graph graph, boolean isChildQuery) { + + return new Titan1GraphQuery(graph, isChildQuery); + } + + /** + * Creates a Titan1Vertex that corresponds to the given Gremlin Vertex. + * + * @param graph The graph that contains the vertex + * @param source the Gremlin vertex + */ + public static Titan1Vertex createVertex(Titan1Graph graph, Vertex source) { + + if (source == null) { + return null; + } + return new Titan1Vertex(graph, source); + } + + /** + * @param propertyKey The Gremlin propertyKey. + * + */ + public static Titan1PropertyKey createPropertyKey(PropertyKey propertyKey) { + if (propertyKey == null) { + return null; + } + return new Titan1PropertyKey(propertyKey); + } + + /** + * @param index The gremlin index. + * @return + */ + public static AtlasGraphIndex createGraphIndex(TitanGraphIndex index) { + if (index == null) { + return null; + } + return new Titan1GraphIndex(index); + } + + /** + * Converts a Multiplicity to a Cardinality. + * + * @param cardinality + * @return + */ + public static AtlasCardinality createCardinality(Cardinality cardinality) { + + if (cardinality == Cardinality.SINGLE) { + return AtlasCardinality.SINGLE; + } else if (cardinality == Cardinality.LIST) { + return AtlasCardinality.LIST; + } + return AtlasCardinality.SET; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1Edge.java ---------------------------------------------------------------------- diff --git a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1Edge.java b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1Edge.java new file mode 100644 index 0000000..bcc009b --- /dev/null +++ b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1Edge.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.graphdb.titan1; +import org.apache.atlas.repository.graphdb.AtlasEdge; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.tinkerpop.gremlin.structure.Edge; + +/** + * Titan 1.0.0 implementation of AtlasEdge. + */ +public class Titan1Edge extends Titan1Element implements AtlasEdge { + + + public Titan1Edge(Titan1Graph graph, Edge edge) { + super(graph, edge); + } + + @Override + public String getLabel() { + return getWrappedElement().label(); + } + + @Override + public Titan1Edge getE() { + + return this; + } + + @Override + public AtlasVertex getInVertex() { + return GraphDbObjectFactory.createVertex(graph, getWrappedElement().inVertex()); + } + + @Override + public AtlasVertex getOutVertex() { + return GraphDbObjectFactory.createVertex(graph, getWrappedElement().outVertex()); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1Element.java ---------------------------------------------------------------------- diff --git a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1Element.java b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1Element.java new file mode 100644 index 0000000..985294e --- /dev/null +++ b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1Element.java @@ -0,0 +1,258 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.graphdb.titan1; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import org.apache.atlas.repository.graphdb.AtlasEdge; +import org.apache.atlas.repository.graphdb.AtlasElement; +import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.graphdb.titan1.graphson.AtlasGraphSONMode; +import org.apache.atlas.repository.graphdb.titan1.graphson.AtlasGraphSONUtility; +import org.apache.tinkerpop.gremlin.structure.Element; +import org.apache.tinkerpop.gremlin.structure.Property; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; + +import com.google.common.base.Function; +import com.google.common.collect.Lists; +import com.thinkaurelius.titan.core.SchemaViolationException; +import com.thinkaurelius.titan.core.TitanElement; + +/** + * Titan 1.0.0 implementation of AtlasElement. + * + * @param the implementation class of the wrapped Titan 1 element + * that is stored. + */ +public class Titan1Element implements AtlasElement { + + + private T element; + protected Titan1Graph graph; + + public Titan1Element(Titan1Graph graph, T element) { + this.element = element; + this.graph = graph; + } + + @Override + public T getProperty(String propertyName, Class clazz) { + + //add explicit logic to return null if the property does not exist + //This is the behavior Atlas expects. Titan 1 throws an exception + //in this scenario. + Property p = getWrappedElement().property(propertyName); + if (p.isPresent()) { + Object propertyValue= p.value(); + if (propertyValue == null) { + return null; + } + if (AtlasEdge.class == clazz) { + return (T)graph.getEdge(propertyValue.toString()); + } + if (AtlasVertex.class == clazz) { + return (T)graph.getVertex(propertyValue.toString()); + } + return (T)propertyValue; + + } + return null; + } + + + + /** + * Gets all of the values of the given property. + * @param propertyName + * @return + */ + @Override + public Collection getPropertyValues(String propertyName, Class type) { + return Collections.singleton(getProperty(propertyName, type)); + } + + @Override + public Set getPropertyKeys() { + return getWrappedElement().keys(); + } + + @Override + public void removeProperty(String propertyName) { + Iterator> it = getWrappedElement().properties(propertyName); + while(it.hasNext()) { + Property property = it.next(); + property.remove(); + } + } + + @Override + public void setProperty(String propertyName, Object value) { + try { + getWrappedElement().property(propertyName, value); + } catch(SchemaViolationException e) { + throw new AtlasSchemaViolationException(e); + } + } + + @Override + public Object getId() { + return element.id(); + } + + + //not in interface + public T getWrappedElement() { + return element; + } + + + @Override + public JSONObject toJson(Set propertyKeys) throws JSONException { + + return AtlasGraphSONUtility.jsonFromElement(this, propertyKeys, AtlasGraphSONMode.NORMAL); + } + + @Override + public int hashCode() { + int result = 37; + result = 17*result + getClass().hashCode(); + result = 17*result + getWrappedElement().hashCode(); + return result; + } + + @Override + public boolean equals(Object other) { + + if (other == null) { + return false; + } + + if (other.getClass() != getClass()) { + return false; + } + Titan1Element otherElement = (Titan1Element) other; + return getWrappedElement().equals(otherElement.getWrappedElement()); + } + + @Override + public List getListProperty(String propertyName) { + List value = getProperty(propertyName, List.class); + if (value == null) { + return Collections.emptyList(); + } + return value; + } + + @Override + public void setListProperty(String propertyName, List values) { + setProperty(propertyName, values); + + } + + @Override + public boolean exists() { + try { + return !((TitanElement)element).isRemoved(); + } catch(IllegalStateException e) { + return false; + } + + } + + @Override + public void setJsonProperty(String propertyName, T value) { + setProperty(propertyName, value); + } + + @Override + public T getJsonProperty(String propertyName) { + return (T)getProperty(propertyName, String.class); + } + + @Override + public String getIdForDisplay() { + return getId().toString(); + } + + + @Override + public List getListProperty(String propertyName, Class elementType) { + + List value = getListProperty(propertyName); + + if (value.isEmpty()) { + return (List)value; + } + + if (AtlasEdge.class.isAssignableFrom(elementType)) { + + + return (List)Lists.transform(value, new Function(){ + + @Override + public AtlasEdge apply(String input) { + return graph.getEdge(input); + } + }); + } + + if (AtlasVertex.class.isAssignableFrom(elementType)) { + + return (List)Lists.transform(value, new Function(){ + + @Override + public AtlasVertex apply(String input) { + return graph.getVertex(input); + } + }); + } + + return (List)value; + } + + + @Override + public void setPropertyFromElementsIds(String propertyName, List values) { + List propertyValue = new ArrayList<>(values.size()); + for(AtlasElement value: values) { + propertyValue.add(value.getId().toString()); + } + setProperty(propertyName, propertyValue); + } + + + @Override + public void setPropertyFromElementId(String propertyName, AtlasElement value) { + setProperty(propertyName, value.getId().toString()); + + } + + + @Override + public boolean isIdAssigned() { + return true; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1Graph.java ---------------------------------------------------------------------- diff --git a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1Graph.java b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1Graph.java new file mode 100644 index 0000000..9a4475d --- /dev/null +++ b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1Graph.java @@ -0,0 +1,405 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.graphdb.titan1; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.script.Bindings; +import javax.script.ScriptEngine; +import javax.script.ScriptException; + +import org.apache.atlas.groovy.GroovyExpression; +import org.apache.atlas.repository.graphdb.AtlasEdge; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasGraphManagement; +import org.apache.atlas.repository.graphdb.AtlasGraphQuery; +import org.apache.atlas.repository.graphdb.AtlasIndexQuery; +import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.graphdb.GremlinVersion; +import org.apache.atlas.repository.graphdb.titan1.query.Titan1GraphQuery; +import org.apache.atlas.repository.graphdb.utils.IteratorToIterableAdapter; +import org.apache.atlas.typesystem.types.IDataType; +import org.apache.tinkerpop.gremlin.groovy.CompilerCustomizerProvider; +import org.apache.tinkerpop.gremlin.groovy.DefaultImportCustomizerProvider; +import org.apache.tinkerpop.gremlin.groovy.jsr223.GremlinGroovyScriptEngine; +import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.ImmutablePath; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Element; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.io.IoCore; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter; + +import com.google.common.base.Function; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.thinkaurelius.titan.core.Cardinality; +import com.thinkaurelius.titan.core.PropertyKey; +import com.thinkaurelius.titan.core.SchemaViolationException; +import com.thinkaurelius.titan.core.TitanGraph; +import com.thinkaurelius.titan.core.TitanIndexQuery; +import com.thinkaurelius.titan.core.schema.TitanGraphIndex; +import com.thinkaurelius.titan.core.schema.TitanManagement; +import com.thinkaurelius.titan.core.util.TitanCleanup; + +/** + * Titan 1.0.0 implementation of AtlasGraph. + */ +public class Titan1Graph implements AtlasGraph { + + private final ConvertGremlinValueFunction GREMLIN_VALUE_CONVERSION_FUNCTION = new ConvertGremlinValueFunction(); + + private final class ConvertGremlinValueFunction implements Function { + @Override + public Object apply(Object input) { + return convertGremlinValue(input); + } + } + + private final Set multiProperties; + + public Titan1Graph() { + //determine multi-properties once at startup + TitanManagement mgmt = null; + try { + mgmt = Titan1GraphDatabase.getGraphInstance().openManagement(); + Iterable keys = mgmt.getRelationTypes(PropertyKey.class); + multiProperties = new HashSet<>(); + for (PropertyKey key : keys) { + if (key.cardinality() != Cardinality.SINGLE) { + multiProperties.add(key.name()); + } + } + } finally { + if (mgmt != null) { + mgmt.rollback(); + } + } + } + + @Override + public AtlasEdge addEdge(AtlasVertex outVertex, + AtlasVertex inVertex, + String edgeLabel) { + + try { + Vertex oV = outVertex.getV().getWrappedElement(); + Vertex iV = inVertex.getV().getWrappedElement(); + Edge edge = oV.addEdge(edgeLabel, iV); + return GraphDbObjectFactory.createEdge(this, edge); + } catch (SchemaViolationException e) { + throw new AtlasSchemaViolationException(e); + } + } + + @Override + public AtlasGraphQuery query() { + return new Titan1GraphQuery(this); + } + + @Override + public AtlasEdge getEdge(String edgeId) { + Iterator it = getGraph().edges(edgeId); + Edge e = getSingleElement(it, edgeId); + return GraphDbObjectFactory.createEdge(this, e); + } + + @Override + public void removeEdge(AtlasEdge edge) { + + Edge wrapped = edge.getE().getWrappedElement(); + wrapped.remove(); + + } + + @Override + public void removeVertex(AtlasVertex vertex) { + Vertex wrapped = vertex.getV().getWrappedElement(); + wrapped.remove(); + } + + @Override + public Iterable> getEdges() { + + Iterator edges = getGraph().edges(); + return wrapEdges(edges); + + } + + @Override + public Iterable> getVertices() { + + Iterator vertices = getGraph().vertices(); + return wrapVertices(vertices); + } + + @Override + public AtlasVertex addVertex() { + Vertex result = getGraph().addVertex(); + return GraphDbObjectFactory.createVertex(this, result); + } + + @Override + public void commit() { + getGraph().tx().commit(); + } + + @Override + public void rollback() { + getGraph().tx().rollback(); + } + + @Override + public AtlasIndexQuery indexQuery(String fulltextIndex, String graphQuery) { + TitanIndexQuery query = getGraph().indexQuery(fulltextIndex, graphQuery); + return new Titan1IndexQuery(this, query); + } + + @Override + public AtlasGraphManagement getManagementSystem() { + return new Titan1GraphManagement(this, getGraph().openManagement()); + } + + @Override + public void shutdown() { + getGraph().close(); + } + + @Override + public Set getEdgeIndexKeys() { + return getIndexKeys(Edge.class); + } + + @Override + public Set getVertexIndexKeys() { + return getIndexKeys(Vertex.class); + } + + private Set getIndexKeys(Class titanElementClass) { + + TitanManagement mgmt = getGraph().openManagement(); + Iterable indices = mgmt.getGraphIndexes(titanElementClass); + Set result = new HashSet(); + for (TitanGraphIndex index : indices) { + result.add(index.name()); + } + mgmt.commit(); + return result; + + } + + @Override + public AtlasVertex getVertex(String vertexId) { + Iterator it = getGraph().vertices(vertexId); + Vertex vertex = getSingleElement(it, vertexId); + return GraphDbObjectFactory.createVertex(this, vertex); + } + + public static T getSingleElement(Iterator it, String id) { + if (!it.hasNext()) { + return null; + } + T element = it.next(); + if (it.hasNext()) { + throw new RuntimeException("Multiple items were found with the id " + id); + } + return element; + } + + @Override + public Iterable> getVertices(String key, Object value) { + AtlasGraphQuery query = query(); + query.has(key, value); + return query.vertices(); + } + + private Object convertGremlinValue(Object rawValue) { + + if (rawValue instanceof Vertex) { + return GraphDbObjectFactory.createVertex(this, (Vertex) rawValue); + } else if (rawValue instanceof Edge) { + return GraphDbObjectFactory.createEdge(this, (Edge) rawValue); + } else if (rawValue instanceof Map) { + Map rowValue = (Map)rawValue; + return Maps.transformValues(rowValue, GREMLIN_VALUE_CONVERSION_FUNCTION); + } else if (rawValue instanceof ImmutablePath) { + ImmutablePath path = (ImmutablePath) rawValue; + return convertGremlinValue(path.objects()); + } + else if (rawValue instanceof List) { + return Lists.transform((List)rawValue, GREMLIN_VALUE_CONVERSION_FUNCTION); + } else if (rawValue instanceof Collection) { + throw new UnsupportedOperationException("Unhandled collection type: " + rawValue.getClass()); + } + return rawValue; + } + + @Override + public GremlinVersion getSupportedGremlinVersion() { + + return GremlinVersion.THREE; + } + @Override + public void clear() { + TitanGraph graph = getGraph(); + if (graph.isOpen()) { + // only a shut down graph can be cleared + graph.close(); + } + TitanCleanup.clear(graph); + } + + private TitanGraph getGraph() { + return Titan1GraphDatabase.getGraphInstance(); + } + + @Override + public void exportToGson(OutputStream os) throws IOException { + + GraphSONMapper mapper = getGraph().io(IoCore.graphson()).mapper().create(); + GraphSONWriter.Builder builder = GraphSONWriter.build(); + builder.mapper(mapper); + GraphSONWriter writer = builder.create(); + writer.writeGraph(os, getGraph()); + } + + @Override + public Object executeGremlinScript(String query, boolean isPath) throws ScriptException { + + Object result = executeGremlinScript(query); + return convertGremlinValue(result); + } + + private Object executeGremlinScript(String gremlinQuery) throws ScriptException { + + Set extraImports = new HashSet(); + extraImports.add(java.util.function.Function.class.getName()); + + Set extraStaticImports = new HashSet(); + extraStaticImports.add(P.class.getName() + ".*"); + extraStaticImports.add(__.class.getName() + ".*"); + CompilerCustomizerProvider provider = new DefaultImportCustomizerProvider(extraImports, extraStaticImports); + + GremlinGroovyScriptEngine scriptEngine = new GremlinGroovyScriptEngine(provider); + try { + Bindings bindings = scriptEngine.createBindings(); + bindings.put("graph", getGraph()); + bindings.put("g", getGraph().traversal()); + Object result = scriptEngine.eval(gremlinQuery, bindings); + return result; + } finally { + try { + scriptEngine.close(); + } catch (Exception e) { + throw new ScriptException(e); + } + } + } + + @Override + public Object executeGremlinScript(ScriptEngine scriptEngine, + Bindings bindings, String query, boolean isPath) + throws ScriptException { + + if(!bindings.containsKey("g")) { + bindings.put("g", getGraph()); + } + Object result = scriptEngine.eval(query, bindings); + return convertGremlinValue(result); + } + + @Override + public GroovyExpression generatePersisentToLogicalConversionExpression(GroovyExpression expr, IDataType type) { + //nothing special needed, value is stored in required type + return expr; + } + + @Override + public boolean isPropertyValueConversionNeeded(IDataType type) { + return false; + } + + @Override + public boolean requiresInitialIndexedPredicate() { + return false; + } + + @Override + public GroovyExpression getInitialIndexedPredicate(GroovyExpression parent) { + return parent; + } + + @Override + public GroovyExpression addOutputTransformationPredicate(GroovyExpression expr, boolean isSelect, boolean isPath) { + return expr; + } + + public Iterable> wrapEdges(Iterator it) { + Iterable iterable = new IteratorToIterableAdapter(it); + return wrapEdges(iterable); + } + + public Iterable> wrapVertices(Iterator it) { + Iterable iterable = new IteratorToIterableAdapter<>(it); + return wrapVertices(iterable); + } + + public Iterable> wrapVertices(Iterable it) { + + return Iterables.transform(it, new Function>() { + + @Override + public AtlasVertex apply(Vertex input) { + return GraphDbObjectFactory.createVertex(Titan1Graph.this, input); + } + }); + + } + + public Iterable> wrapEdges(Iterable it) { + Iterable result = (Iterable) it; + return Iterables.transform(result, new Function>() { + + @Override + public AtlasEdge apply(Edge input) { + return GraphDbObjectFactory.createEdge(Titan1Graph.this, input); + } + }); + } + + @Override + public boolean isMultiProperty(String propertyName) { + return multiProperties.contains(propertyName); + } + + public void addMultiProperties(Set names) { + multiProperties.addAll(names); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1GraphDatabase.java ---------------------------------------------------------------------- diff --git a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1GraphDatabase.java b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1GraphDatabase.java new file mode 100644 index 0000000..dfb5354 --- /dev/null +++ b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1GraphDatabase.java @@ -0,0 +1,176 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.graphdb.titan1; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.ArrayList; + +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasException; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.GraphDatabase; +import org.apache.atlas.repository.graphdb.titan1.serializer.BigDecimalSerializer; +import org.apache.atlas.repository.graphdb.titan1.serializer.BigIntegerSerializer; +import org.apache.atlas.repository.graphdb.titan1.serializer.StringListSerializer; +import org.apache.atlas.repository.graphdb.titan1.serializer.TypeCategorySerializer; +import org.apache.atlas.typesystem.types.DataTypes.TypeCategory; +import org.apache.commons.configuration.Configuration; +import org.apache.tinkerpop.gremlin.groovy.loaders.SugarLoader; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.thinkaurelius.titan.core.TitanFactory; +import com.thinkaurelius.titan.core.TitanGraph; +import com.thinkaurelius.titan.core.schema.TitanManagement; +import com.thinkaurelius.titan.core.util.TitanCleanup; +import com.thinkaurelius.titan.graphdb.tinkerpop.TitanIoRegistry; + +/** + * Default implementation for Graph Provider that doles out Titan Graph. + */ +public class Titan1GraphDatabase implements GraphDatabase { + + private static final Logger LOG = LoggerFactory.getLogger(Titan1GraphDatabase.class); + + /** + * Constant for the configuration property that indicates the prefix. + */ + public static final String GRAPH_PREFIX = "atlas.graph"; + + public static final String INDEX_BACKEND_CONF = "index.search.backend"; + + public static final String INDEX_BACKEND_LUCENE = "lucene"; + + public static final String INDEX_BACKEND_ES = "elasticsearch"; + + private static volatile Titan1Graph atlasGraphInstance = null; + private static volatile TitanGraph graphInstance; + + public Titan1GraphDatabase() { + + //update registry + GraphSONMapper.build().addRegistry(TitanIoRegistry.INSTANCE).create(); + } + + public static Configuration getConfiguration() throws AtlasException { + Configuration configProperties = ApplicationProperties.get(); + + Configuration titanConfig = ApplicationProperties.getSubsetConfiguration(configProperties, GRAPH_PREFIX); + + //add serializers for non-standard property value types that Atlas uses + + titanConfig.addProperty("attributes.custom.attribute1.attribute-class", TypeCategory.class.getName()); + titanConfig.addProperty("attributes.custom.attribute1.serializer-class", + TypeCategorySerializer.class.getName()); + + //not ideal, but avoids making large changes to Atlas + titanConfig.addProperty("attributes.custom.attribute2.attribute-class", ArrayList.class.getName()); + titanConfig.addProperty("attributes.custom.attribute2.serializer-class", StringListSerializer.class.getName()); + + titanConfig.addProperty("attributes.custom.attribute3.attribute-class", BigInteger.class.getName()); + titanConfig.addProperty("attributes.custom.attribute3.serializer-class", BigIntegerSerializer.class.getName()); + + titanConfig.addProperty("attributes.custom.attribute4.attribute-class", BigDecimal.class.getName()); + titanConfig.addProperty("attributes.custom.attribute4.serializer-class", BigDecimalSerializer.class.getName()); + + return titanConfig; + } + + public static TitanGraph getGraphInstance() { + if (graphInstance == null) { + synchronized (Titan1GraphDatabase.class) { + if (graphInstance == null) { + Configuration config; + try { + config = getConfiguration(); + } catch (AtlasException e) { + throw new RuntimeException(e); + } + + graphInstance = TitanFactory.open(config); + atlasGraphInstance = new Titan1Graph(); + validateIndexBackend(config); + } + } + } + return graphInstance; + } + + public static void unload() { + synchronized (Titan1GraphDatabase.class) { + + if (graphInstance == null) { + return; + } + graphInstance.tx().commit(); + graphInstance.close(); + graphInstance = null; + } + } + + static void validateIndexBackend(Configuration config) { + String configuredIndexBackend = config.getString(INDEX_BACKEND_CONF); + + TitanManagement managementSystem = getGraphInstance().openManagement(); + String currentIndexBackend = managementSystem.get(INDEX_BACKEND_CONF); + managementSystem.commit(); + + if (!configuredIndexBackend.equals(currentIndexBackend)) { + throw new RuntimeException("Configured Index Backend " + configuredIndexBackend + + " differs from earlier configured Index Backend " + currentIndexBackend + ". Aborting!"); + } + + } + + @Override + public boolean isGraphLoaded() { + return graphInstance != null; + } + + @Override + public void initializeTestGraph() { + //nothing to do + + } + + @Override + public void cleanup() { + try { + getGraphInstance().close(); + } catch (Throwable t) { + LOG.warn("Could not close test TitanGraph", t); + t.printStackTrace(); + } + + try { + TitanCleanup.clear(getGraphInstance()); + } catch (Throwable t) { + LOG.warn("Could not clear test TitanGraph", t); + t.printStackTrace(); + } + } + + @Override + public AtlasGraph getGraph() { + getGraphInstance(); + return atlasGraphInstance; + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1GraphIndex.java ---------------------------------------------------------------------- diff --git a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1GraphIndex.java b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1GraphIndex.java new file mode 100644 index 0000000..5ec1180 --- /dev/null +++ b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1GraphIndex.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.graphdb.titan1; + +import java.util.HashSet; +import java.util.Set; + +import org.apache.atlas.repository.graphdb.AtlasGraphIndex; +import org.apache.atlas.repository.graphdb.AtlasPropertyKey; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Vertex; + +import com.thinkaurelius.titan.core.PropertyKey; +import com.thinkaurelius.titan.core.schema.TitanGraphIndex; + +/** + * Represents an Index in Titan 1. + */ +public class Titan1GraphIndex implements AtlasGraphIndex { + + private TitanGraphIndex wrapped; + + public Titan1GraphIndex(TitanGraphIndex toWrap) { + this.wrapped = toWrap; + } + + + @Override + public boolean isEdgeIndex() { + return Edge.class.isAssignableFrom(wrapped.getIndexedElement()); + } + + @Override + public boolean isVertexIndex() { + return Vertex.class.isAssignableFrom(wrapped.getIndexedElement()); + } + + @Override + public boolean isUnique() { + return wrapped.isUnique(); + } + + + @Override + public Set getFieldKeys() { + PropertyKey[] keys = wrapped.getFieldKeys(); + Set result = new HashSet(); + for(PropertyKey key : keys) { + result.add(GraphDbObjectFactory.createPropertyKey(key)); + } + return result; + } + + @Override + public int hashCode() { + int result = 17; + result = 37*result + wrapped.hashCode(); + return result; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Titan1GraphIndex)) { + return false; + } + Titan1GraphIndex otherKey = (Titan1GraphIndex)other; + return otherKey.wrapped.equals(wrapped); + + } + + + @Override + public boolean isMixedIndex() { + return wrapped.isMixedIndex(); + } + + + @Override + public boolean isCompositeIndex() { + return wrapped.isCompositeIndex(); + } + + +}