tinkerpop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From spmalle...@apache.org
Subject [33/50] [abbrv] tinkerpop git commit: Improved memory usage when deserializing TinkerGraph from GraphSON
Date Fri, 26 May 2017 12:59:24 GMT
Improved memory usage when deserializing TinkerGraph from GraphSON

Streamed vertices/edges rather than reading them all into memory at once. CTR


Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/7ea38751
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/7ea38751
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/7ea38751

Branch: refs/heads/TINKERPOP-1489
Commit: 7ea38751871f6eddd4c4bcfe8ac82300a08f5a7d
Parents: afa3432
Author: Stephen Mallette <spmva@genoprime.com>
Authored: Tue May 23 11:01:55 2017 -0400
Committer: Stephen Mallette <spmva@genoprime.com>
Committed: Tue May 23 11:01:55 2017 -0400

----------------------------------------------------------------------
 CHANGELOG.asciidoc                              |  1 +
 .../structure/TinkerIoRegistryV2d0.java         | 40 +++++++++-----------
 2 files changed, 18 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7ea38751/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 7057aac..be8059d 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,7 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima
 TinkerPop 3.2.5 (Release Date: NOT OFFICIALLY RELEASED YET)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+* Reduced memory usage for TinkerGraph deserialization in GraphSON by streaming vertices
and edges.
 * Now using Groovy `[...]` map notation in `GroovyTranslator` instead of `new LinkedHashMap(){{
}}`.
 * Maintained type information on `Traversal.promise()`.
 * Propagated exception to `Future` instead of calling thread in `RemoteConnection`.

http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7ea38751/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerIoRegistryV2d0.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerIoRegistryV2d0.java
b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerIoRegistryV2d0.java
index 727931c..394216b 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerIoRegistryV2d0.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerIoRegistryV2d0.java
@@ -26,7 +26,6 @@ import org.apache.tinkerpop.gremlin.structure.io.AbstractIoRegistry;
 import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
 import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONIo;
 import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONTokens;
-import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONUtil;
 import org.apache.tinkerpop.gremlin.structure.io.graphson.TinkerPopJacksonModule;
 import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
 import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader;
@@ -37,12 +36,11 @@ import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex;
 import org.apache.tinkerpop.shaded.jackson.core.JsonGenerator;
 import org.apache.tinkerpop.shaded.jackson.core.JsonParser;
 import org.apache.tinkerpop.shaded.jackson.core.JsonProcessingException;
+import org.apache.tinkerpop.shaded.jackson.core.JsonToken;
 import org.apache.tinkerpop.shaded.jackson.databind.DeserializationContext;
 import org.apache.tinkerpop.shaded.jackson.databind.SerializerProvider;
 import org.apache.tinkerpop.shaded.jackson.databind.deser.std.StdDeserializer;
-import org.apache.tinkerpop.shaded.jackson.databind.jsontype.TypeSerializer;
 import org.apache.tinkerpop.shaded.jackson.databind.ser.std.StdScalarSerializer;
-import org.apache.tinkerpop.shaded.jackson.databind.ser.std.StdSerializer;
 import org.apache.tinkerpop.shaded.kryo.Kryo;
 import org.apache.tinkerpop.shaded.kryo.Serializer;
 import org.apache.tinkerpop.shaded.kryo.io.Input;
@@ -53,7 +51,6 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Iterator;
-import java.util.List;
 import java.util.Map;
 
 /**
@@ -201,26 +198,23 @@ public final class TinkerIoRegistryV2d0 extends AbstractIoRegistry {
             conf.setProperty("gremlin.tinkergraph.defaultVertexPropertyCardinality", "list");
             final TinkerGraph graph = TinkerGraph.open(conf);
 
-            final List<? extends Edge> edges;
-            final List<? extends Vertex> vertices;
-
-            jsonParser.nextToken();
-            final Map<String, Object> graphData = deserializationContext.readValue(jsonParser,
Map.class);
-            vertices = (List<DetachedVertex>) graphData.get(GraphSONTokens.VERTICES);
-            edges = (List<DetachedEdge>) graphData.get(GraphSONTokens.EDGES);
-
-
-            vertices.forEach(e -> {
-                if (e instanceof DetachedVertex) {
-                    ((DetachedVertex)e).attach(Attachable.Method.getOrCreate(graph));
+            while (jsonParser.nextToken() != JsonToken.END_OBJECT) {
+                if (jsonParser.getCurrentName().equals("vertices")) {
+                    while (jsonParser.nextToken() != JsonToken.END_ARRAY) {
+                        if (jsonParser.currentToken() == JsonToken.START_OBJECT) {
+                            final DetachedVertex v = (DetachedVertex) deserializationContext.readValue(jsonParser,
Vertex.class);
+                            v.attach(Attachable.Method.getOrCreate(graph));
+                        }
+                    }
+                } else if (jsonParser.getCurrentName().equals("edges")) {
+                    while (jsonParser.nextToken() != JsonToken.END_ARRAY) {
+                        if (jsonParser.currentToken() == JsonToken.START_OBJECT) {
+                            final DetachedEdge e = (DetachedEdge) deserializationContext.readValue(jsonParser,
Edge.class);
+                            e.attach(Attachable.Method.getOrCreate(graph));
+                        }
+                    }
                 }
-            });
-
-            edges.forEach(e -> {
-                if (e instanceof DetachedEdge) {
-                    ((DetachedEdge) e).attach(Attachable.Method.getOrCreate(graph));
-                }
-            });
+            }
 
             return graph;
         }


Mime
View raw message