crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject crunch git commit: CRUNCH-544: Improve performance/serializability of materialized toMap.
Date Sat, 18 Jul 2015 15:58:56 GMT
Repository: crunch
Updated Branches:
  refs/heads/master a670b9169 -> b6accf4e3


CRUNCH-544: Improve performance/serializability of materialized toMap.


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/b6accf4e
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/b6accf4e
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/b6accf4e

Branch: refs/heads/master
Commit: b6accf4e33d8311225cf03f29c674dc25bec451e
Parents: a670b91
Author: Josh Wills <jwills@apache.org>
Authored: Fri Jul 17 08:12:09 2015 -0700
Committer: Josh Wills <jwills@apache.org>
Committed: Sat Jul 18 08:46:48 2015 -0700

----------------------------------------------------------------------
 .../org/apache/crunch/MaterializeToMapIT.java   |  4 ++
 .../crunch/materialize/MaterializableMap.java   | 43 +++++++++++++++-----
 2 files changed, 36 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/b6accf4e/crunch-core/src/it/java/org/apache/crunch/MaterializeToMapIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/MaterializeToMapIT.java b/crunch-core/src/it/java/org/apache/crunch/MaterializeToMapIT.java
index 8457bac..d65b708 100644
--- a/crunch-core/src/it/java/org/apache/crunch/MaterializeToMapIT.java
+++ b/crunch-core/src/it/java/org/apache/crunch/MaterializeToMapIT.java
@@ -20,8 +20,10 @@ package org.apache.crunch;
 import static org.junit.Assert.assertEquals;
 
 import java.io.IOException;
+import java.io.Serializable;
 import java.util.Map;
 
+import org.apache.commons.lang.SerializationUtils;
 import org.apache.crunch.impl.mem.MemPipeline;
 import org.apache.crunch.impl.mr.MRPipeline;
 import org.apache.crunch.test.TemporaryPath;
@@ -77,6 +79,8 @@ public class MaterializeToMapIT {
     PTable<Integer, String> t = c.parallelDo(new Set1Mapper(), tf.tableOf(tf.ints(),
tf.strings()));
     Map<Integer, String> m = t.materializeToMap();
     assertMatches(m);
+    Map<Integer, String> mclone = (Map<Integer, String>) SerializationUtils.clone((Serializable)
m);
+    assertMatches(mclone);
   }
 
 }

http://git-wip-us.apache.org/repos/asf/crunch/blob/b6accf4e/crunch-core/src/main/java/org/apache/crunch/materialize/MaterializableMap.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/materialize/MaterializableMap.java
b/crunch-core/src/main/java/org/apache/crunch/materialize/MaterializableMap.java
index 69082e2..d8c98d0 100644
--- a/crunch-core/src/main/java/org/apache/crunch/materialize/MaterializableMap.java
+++ b/crunch-core/src/main/java/org/apache/crunch/materialize/MaterializableMap.java
@@ -17,6 +17,7 @@
  */
 package org.apache.crunch.materialize;
 
+import java.io.Serializable;
 import java.util.AbstractMap;
 import java.util.HashMap;
 import java.util.Map;
@@ -24,27 +25,47 @@ import java.util.Set;
 
 import org.apache.crunch.Pair;
 
-public class MaterializableMap<K, V> extends AbstractMap<K, V> {
+public class MaterializableMap<K, V> extends AbstractMap<K, V> implements Serializable
{
 
-  private Iterable<Pair<K, V>> iterable;
-  private Set<Map.Entry<K, V>> entrySet;
+  private transient Iterable<Pair<K, V>> iterable;
+  private Map<K, V> delegate;
 
   public MaterializableMap(Iterable<Pair<K, V>> iterable) {
     this.iterable = iterable;
   }
 
-  private Set<Map.Entry<K, V>> toMapEntries(Iterable<Pair<K, V>>
xs) {
-    HashMap<K, V> m = new HashMap<K, V>();
-    for (Pair<K, V> x : xs)
-      m.put(x.first(), x.second());
-    return m.entrySet();
+  private Map<K, V> delegate() {
+    if (delegate == null) {
+      delegate = new HashMap<K, V>();
+      for (Pair<K, V> x : iterable) {
+        delegate.put(x.first(), x.second());
+      }
+    }
+    return delegate;
   }
 
   @Override
   public Set<Map.Entry<K, V>> entrySet() {
-    if (entrySet == null)
-      entrySet = toMapEntries(iterable);
-    return entrySet;
+    return delegate().entrySet();
   }
 
+  @Override
+  public V get(Object key) {
+    return delegate().get(key);
+  }
+
+  @Override
+  public boolean containsKey(Object key) {
+    return delegate().containsKey(key);
+  }
+
+  @Override
+  public int hashCode() {
+    return delegate().hashCode();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    return delegate().equals(other);
+  }
 }
\ No newline at end of file


Mime
View raw message