Return-Path: X-Original-To: apmail-atlas-commits-archive@minotaur.apache.org Delivered-To: apmail-atlas-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 9EADA17B72 for ; Wed, 13 May 2015 21:29:20 +0000 (UTC) Received: (qmail 80841 invoked by uid 500); 13 May 2015 21:29:20 -0000 Delivered-To: apmail-atlas-commits-archive@atlas.apache.org Received: (qmail 80822 invoked by uid 500); 13 May 2015 21:29:20 -0000 Mailing-List: contact commits-help@atlas.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@atlas.incubator.apache.org Delivered-To: mailing list commits@atlas.incubator.apache.org Received: (qmail 80813 invoked by uid 99); 13 May 2015 21:29:20 -0000 Received: from Unknown (HELO spamd1-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 13 May 2015 21:29:20 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd1-us-west.apache.org (ASF Mail Server at spamd1-us-west.apache.org) with ESMTP id 1CEFCC491F for ; Wed, 13 May 2015 21:29:20 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd1-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: 1.771 X-Spam-Level: * X-Spam-Status: No, score=1.771 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, T_RP_MATCHES_RCVD=-0.01, URIBL_BLOCKED=0.001] autolearn=disabled Received: from mx1-us-west.apache.org ([10.40.0.8]) by localhost (spamd1-us-west.apache.org [10.40.0.7]) (amavisd-new, port 10024) with ESMTP id 0_OpPBxO7CVD for ; Wed, 13 May 2015 21:29:16 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-us-west.apache.org (ASF Mail Server at mx1-us-west.apache.org) with SMTP id C9207286CB for ; Wed, 13 May 2015 21:28:59 +0000 (UTC) Received: (qmail 77723 invoked by uid 99); 13 May 2015 21:27:44 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 13 May 2015 21:27:44 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A670DE35B7; Wed, 13 May 2015 21:27:44 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: venkatesh@apache.org To: commits@atlas.incubator.apache.org Date: Wed, 13 May 2015 21:28:17 -0000 Message-Id: <091ba8f4732a443c8209c2e7e5b05f59@git.apache.org> In-Reply-To: <5b1cdabdc6644cd6bfb5693f3ca1e58c@git.apache.org> References: <5b1cdabdc6644cd6bfb5693f3ca1e58c@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [36/50] [abbrv] incubator-atlas git commit: fixed test failures - GraphBackedMetadataRepositoryTest#testFullTextSearch, DSLTest.test1 fixed test failures - GraphBackedMetadataRepositoryTest#testFullTextSearch, DSLTest.test1 Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/4b2a4cae Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/4b2a4cae Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/4b2a4cae Branch: refs/remotes/origin/master Commit: 4b2a4caed572fb358cddb37ce1b183072e08a0a6 Parents: 8e73ed2 Author: Shwetha GS Authored: Mon May 4 14:59:11 2015 +0530 Committer: Shwetha GS Committed: Mon May 4 14:59:22 2015 +0530 ---------------------------------------------------------------------- .../graph/GraphBackedMetadataRepository.java | 71 +++++++++----------- .../metadata/repository/graph/GraphHelper.java | 2 +- .../GraphBackedMetadataRepositoryTest.java | 14 ++-- .../hadoop/metadata/tools/dsl/package.scala | 2 +- .../hadoop/metadata/tools/dsl/DSLTest.scala | 2 +- .../metadata/typesystem/types/TypeSystem.java | 8 +-- 6 files changed, 44 insertions(+), 55 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4b2a4cae/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java index d6a3123..d273787 100755 --- a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java +++ b/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java @@ -473,9 +473,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { Id id = typedInstance.getId(); Vertex instanceVertex = entityProcessor.idToVertexMap.get(id); String fullText = getFullTextForVertex(instanceVertex, true); - instanceVertex.setProperty(Constants.ENTITY_TEXT_PROPERTY_KEY, fullText); - LOG.debug("Adding {} for {} = {}", Constants.ENTITY_TEXT_PROPERTY_KEY, - instanceVertex, fullText); + addProperty(instanceVertex, Constants.ENTITY_TEXT_PROPERTY_KEY, fullText); } } @@ -658,8 +656,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository { private void mapInstanceToVertex(Id id, ITypedInstance typedInstance, Vertex instanceVertex, Map fields, Map idToVertexMap) throws MetadataException { - LOG.debug("Mapping instance {} to vertex {} for fields {}", - typedInstance.getTypeName(), instanceVertex, fields); + LOG.debug("Mapping instance {} of {} to vertex {}", + typedInstance, typedInstance.getTypeName(), instanceVertex); for (AttributeInfo attributeInfo : fields.values()) { final IDataType dataType = attributeInfo.dataType(); mapAttributesToVertex(id, typedInstance, instanceVertex, @@ -672,9 +670,10 @@ public class GraphBackedMetadataRepository implements MetadataRepository { Map idToVertexMap, AttributeInfo attributeInfo, IDataType dataType) throws MetadataException { - LOG.debug("mapping attributeInfo {}", attributeInfo); + Object attrValue = typedInstance.get(attributeInfo.name); + LOG.debug("mapping attribute {} = {}", attributeInfo.name, attrValue); final String propertyName = getQualifiedName(typedInstance, attributeInfo); - if (typedInstance.get(attributeInfo.name) == null) { + if (attrValue == null) { return; } @@ -684,8 +683,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { break; case ENUM: - instanceVertex.setProperty(propertyName, - typedInstance.getInt(attributeInfo.name)); + addProperty(instanceVertex, propertyName, typedInstance.getInt(attributeInfo.name)); break; case ARRAY: @@ -749,7 +747,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { buffer.setLength(buffer.length() - 1); // for dereference on way out - instanceVertex.setProperty(propertyName, buffer.toString()); + addProperty(instanceVertex, propertyName, buffer.toString()); } private void mapMapCollectionToVertex(Id id, ITypedInstance typedInstance, @@ -778,7 +776,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { buffer.setLength(buffer.length() - 1); // for dereference on way out - instanceVertex.setProperty(propertyName, buffer.toString()); + addProperty(instanceVertex, propertyName, buffer.toString()); } private String mapCollectionEntryToVertex(Id id, Vertex instanceVertex, @@ -792,7 +790,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { switch (elementType.getTypeCategory()) { case PRIMITIVE: case ENUM: - instanceVertex.setProperty(propertyNameWithSuffix, value); + addProperty(instanceVertex, propertyNameWithSuffix, value); return propertyNameWithSuffix; case ARRAY: @@ -851,7 +849,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository { Vertex structInstanceVertex = GraphHelper.createVertexWithoutIdentity( titanGraph, structInstance.getTypeName(), id, Collections.emptySet()); // no super types for struct type - LOG.debug("created vertex {} for struct {}", structInstanceVertex, attributeInfo.name); + LOG.debug("created vertex {} for struct {} value {}", structInstanceVertex, attributeInfo.name, + structInstance); // map all the attributes to this newly created vertex mapInstanceToVertex(id, structInstance, structInstanceVertex, @@ -895,47 +894,43 @@ public class GraphBackedMetadataRepository implements MetadataRepository { private void mapPrimitiveToVertex(ITypedInstance typedInstance, Vertex instanceVertex, AttributeInfo attributeInfo) throws MetadataException { - LOG.debug("Adding primitive {} to v {}", attributeInfo, instanceVertex); - if (typedInstance.get(attributeInfo.name) == null) { + Object attrValue = typedInstance.get(attributeInfo.name); + if (attrValue == null) { return; // add only if instance has this attribute } final String vertexPropertyName = getQualifiedName(typedInstance, attributeInfo); - + Object propertyValue = null; if (attributeInfo.dataType() == DataTypes.STRING_TYPE) { - instanceVertex.setProperty(vertexPropertyName, - typedInstance.getString(attributeInfo.name)); + propertyValue = typedInstance.getString(attributeInfo.name); } else if (attributeInfo.dataType() == DataTypes.SHORT_TYPE) { - instanceVertex.setProperty(vertexPropertyName, - typedInstance.getShort(attributeInfo.name)); + propertyValue = typedInstance.getShort(attributeInfo.name); } else if (attributeInfo.dataType() == DataTypes.INT_TYPE) { - instanceVertex.setProperty(vertexPropertyName, - typedInstance.getInt(attributeInfo.name)); + propertyValue = typedInstance.getInt(attributeInfo.name); } else if (attributeInfo.dataType() == DataTypes.BIGINTEGER_TYPE) { - instanceVertex.setProperty(vertexPropertyName, - typedInstance.getBigInt(attributeInfo.name)); + propertyValue = typedInstance.getBigInt(attributeInfo.name); } else if (attributeInfo.dataType() == DataTypes.BOOLEAN_TYPE) { - instanceVertex.setProperty(vertexPropertyName, - typedInstance.getBoolean(attributeInfo.name)); + propertyValue = typedInstance.getBoolean(attributeInfo.name); } else if (attributeInfo.dataType() == DataTypes.BYTE_TYPE) { - instanceVertex.setProperty(vertexPropertyName, - typedInstance.getByte(attributeInfo.name)); + propertyValue = typedInstance.getByte(attributeInfo.name); } else if (attributeInfo.dataType() == DataTypes.LONG_TYPE) { - instanceVertex.setProperty(vertexPropertyName, - typedInstance.getLong(attributeInfo.name)); + propertyValue = typedInstance.getLong(attributeInfo.name); } else if (attributeInfo.dataType() == DataTypes.FLOAT_TYPE) { - instanceVertex.setProperty(vertexPropertyName, - typedInstance.getFloat(attributeInfo.name)); + propertyValue = typedInstance.getFloat(attributeInfo.name); } else if (attributeInfo.dataType() == DataTypes.DOUBLE_TYPE) { - instanceVertex.setProperty(vertexPropertyName, - typedInstance.getDouble(attributeInfo.name)); + propertyValue = typedInstance.getDouble(attributeInfo.name); } else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) { - instanceVertex.setProperty(vertexPropertyName, - typedInstance.getBigDecimal(attributeInfo.name)); + propertyValue = typedInstance.getBigDecimal(attributeInfo.name); } + addProperty(instanceVertex, vertexPropertyName, propertyValue); } } + private void addProperty(Vertex vertex, String propertyName, Object value) { + LOG.debug("Setting property {} = \"{}\" to vertex {}", propertyName, value, vertex); + vertex.setProperty(propertyName, value); + } + public final class GraphToTypedInstanceMapper { public ITypedReferenceableInstance mapGraphToTypedInstance(String guid, @@ -985,7 +980,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { public void mapVertexToAttribute(Vertex instanceVertex, ITypedInstance typedInstance, AttributeInfo attributeInfo) throws MetadataException { - LOG.debug("mapping attributeInfo = " + attributeInfo); + LOG.debug("mapping attributeInfo {}", attributeInfo.name); final IDataType dataType = attributeInfo.dataType(); final String vertexPropertyName = getQualifiedName(typedInstance, attributeInfo); @@ -1226,7 +1221,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ITypedStruct structInstance = structType.createInstance(); typedInstance.set(attributeInfo.name, structInstance); - String relationshipLabel = getQualifiedName(structType, attributeInfo.name); + String relationshipLabel = getQualifiedName(typedInstance, attributeInfo); LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel); for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) { final Vertex structInstanceVertex = edge.getVertex(Direction.IN); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4b2a4cae/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java index 970bd81..ac9e2fa 100755 --- a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java @@ -85,7 +85,7 @@ public final class GraphHelper { public static Edge addEdge(TitanGraph titanGraph, Vertex fromVertex, Vertex toVertex, String edgeLabel) { - LOG.debug("Adding edge for {} -> struct label {} -> v{}", + LOG.debug("Adding edge for {} -> label {} -> {}", fromVertex, edgeLabel, toVertex); return titanGraph.addEdge(null, fromVertex, toVertex, edgeLabel); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4b2a4cae/repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java b/repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java index 89609db..842677b 100755 --- a/repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java +++ b/repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java @@ -334,11 +334,6 @@ public class GraphBackedMetadataRepositoryTest { String response = discoveryService.searchByFullText("john"); Assert.assertNotNull(response); JSONArray results = new JSONArray(response); - System.out.println("Found the following results"); - for (int i = 0 ; i < results.length(); i++) { - JSONObject myrow = results.getJSONObject(i); - System.out.println(myrow.toString()); - } Assert.assertEquals(results.length(), 1); JSONObject row = (JSONObject) results.get(0); Assert.assertEquals(row.get("typeName"), "Person"); @@ -346,11 +341,10 @@ public class GraphBackedMetadataRepositoryTest { //person in hr department who lives in santa clara response = discoveryService.searchByFullText("Jane AND santa AND clara"); Assert.assertNotNull(response); - // todo: enable this - temporarily commented this as its failing -// results = new JSONArray(response); -// Assert.assertEquals(results.length(), 1); -// row = (JSONObject) results.get(0); -// Assert.assertEquals(row.get("typeName"), "Manager"); + results = new JSONArray(response); + Assert.assertEquals(results.length(), 1); + row = (JSONObject) results.get(0); + Assert.assertEquals(row.get("typeName"), "Manager"); //search for person in hr department whose name starts is john/jahn response = discoveryService.searchByFullText("hr AND (john OR jahn)"); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4b2a4cae/tools/src/main/scala/org/apache/hadoop/metadata/tools/dsl/package.scala ---------------------------------------------------------------------- diff --git a/tools/src/main/scala/org/apache/hadoop/metadata/tools/dsl/package.scala b/tools/src/main/scala/org/apache/hadoop/metadata/tools/dsl/package.scala index de62551..ba09342 100755 --- a/tools/src/main/scala/org/apache/hadoop/metadata/tools/dsl/package.scala +++ b/tools/src/main/scala/org/apache/hadoop/metadata/tools/dsl/package.scala @@ -36,7 +36,7 @@ import scala.collection.JavaConversions._ package object dsl { val defFormat = new DefaultFormats { - override protected def dateFormatter = new SimpleDateFormat("yyyy-MM-dd") + override protected def dateFormatter = TypeSystem.getInstance().getDateFormat; override val typeHints = NoTypeHints } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4b2a4cae/tools/src/test/scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala ---------------------------------------------------------------------- diff --git a/tools/src/test/scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala b/tools/src/test/scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala index 47db8fe..1698d40 100755 --- a/tools/src/test/scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala +++ b/tools/src/test/scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala @@ -120,7 +120,7 @@ class DSLTest { Assert.assertEquals(s"${i.o.asInstanceOf[java.util.Map[_, _]].keySet}", "[b, a]") // 5. Serialize mytype instance to Json - Assert.assertEquals(s"${pretty(render(i))}", "{\n \"$typeName$\":\"mytype\",\n \"e\":1," + "\n \"n\":[1,1.100000000000000088817841970012523233890533447265625],\n \"h\":1.0,\n \"b\":true,\n \"k\":1,\n \"j\":1,\n \"d\":2,\n \"m\":[1,1],\n \"g\":1,\n \"a\":1,\n \"i\":1.0,\n \"c\":1,\n \"l\":\"2014-12-02\",\n \"f\":1,\n \"o\":{\n \"b\":2.0,\n \"a\":1.0\n }\n}") + Assert.assertEquals(s"${pretty(render(i))}", "{\n \"$typeName$\":\"mytype\",\n \"e\":1," + "\n \"n\":[1,1.100000000000000088817841970012523233890533447265625],\n \"h\":1.0,\n \"b\":true,\n \"k\":1,\n \"j\":1,\n \"d\":2,\n \"m\":[1,1],\n \"g\":1,\n \"a\":1,\n \"i\":1.0,\n \"c\":1,\n \"l\":\"2014-12-03\",\n \"f\":1,\n \"o\":{\n \"b\":2.0,\n \"a\":1.0\n }\n}") } @Test def test2 { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4b2a4cae/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java index c04016e..61c60dc 100755 --- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java +++ b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java @@ -42,10 +42,10 @@ import java.util.concurrent.ConcurrentHashMap; @InterfaceAudience.Private public class TypeSystem { private static final TypeSystem INSTANCE = new TypeSystem(); - public static ThreadLocal dateFormat = new ThreadLocal() { + private static ThreadLocal dateFormat = new ThreadLocal() { @Override - public DateFormat initialValue() { - DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + public SimpleDateFormat initialValue() { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); return dateFormat; } @@ -294,7 +294,7 @@ public class TypeSystem { return eT; } - public DateFormat getDateFormat() { + public SimpleDateFormat getDateFormat() { return dateFormat.get(); }