Return-Path: X-Original-To: apmail-crunch-commits-archive@www.apache.org Delivered-To: apmail-crunch-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id ED07B18DB5 for ; Sat, 18 Jul 2015 15:58:56 +0000 (UTC) Received: (qmail 13479 invoked by uid 500); 18 Jul 2015 15:58:56 -0000 Delivered-To: apmail-crunch-commits-archive@crunch.apache.org Received: (qmail 13444 invoked by uid 500); 18 Jul 2015 15:58:56 -0000 Mailing-List: contact commits-help@crunch.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@crunch.apache.org Delivered-To: mailing list commits@crunch.apache.org Received: (qmail 13435 invoked by uid 99); 18 Jul 2015 15:58:56 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 18 Jul 2015 15:58:56 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 9A88DE0941; Sat, 18 Jul 2015 15:58:56 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jwills@apache.org To: commits@crunch.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: crunch git commit: CRUNCH-544: Improve performance/serializability of materialized toMap. Date: Sat, 18 Jul 2015 15:58:56 +0000 (UTC) Repository: crunch Updated Branches: refs/heads/master a670b9169 -> b6accf4e3 CRUNCH-544: Improve performance/serializability of materialized toMap. Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/b6accf4e Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/b6accf4e Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/b6accf4e Branch: refs/heads/master Commit: b6accf4e33d8311225cf03f29c674dc25bec451e Parents: a670b91 Author: Josh Wills Authored: Fri Jul 17 08:12:09 2015 -0700 Committer: Josh Wills Committed: Sat Jul 18 08:46:48 2015 -0700 ---------------------------------------------------------------------- .../org/apache/crunch/MaterializeToMapIT.java | 4 ++ .../crunch/materialize/MaterializableMap.java | 43 +++++++++++++++----- 2 files changed, 36 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/b6accf4e/crunch-core/src/it/java/org/apache/crunch/MaterializeToMapIT.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/it/java/org/apache/crunch/MaterializeToMapIT.java b/crunch-core/src/it/java/org/apache/crunch/MaterializeToMapIT.java index 8457bac..d65b708 100644 --- a/crunch-core/src/it/java/org/apache/crunch/MaterializeToMapIT.java +++ b/crunch-core/src/it/java/org/apache/crunch/MaterializeToMapIT.java @@ -20,8 +20,10 @@ package org.apache.crunch; import static org.junit.Assert.assertEquals; import java.io.IOException; +import java.io.Serializable; import java.util.Map; +import org.apache.commons.lang.SerializationUtils; import org.apache.crunch.impl.mem.MemPipeline; import org.apache.crunch.impl.mr.MRPipeline; import org.apache.crunch.test.TemporaryPath; @@ -77,6 +79,8 @@ public class MaterializeToMapIT { PTable t = c.parallelDo(new Set1Mapper(), tf.tableOf(tf.ints(), tf.strings())); Map m = t.materializeToMap(); assertMatches(m); + Map mclone = (Map) SerializationUtils.clone((Serializable) m); + assertMatches(mclone); } } http://git-wip-us.apache.org/repos/asf/crunch/blob/b6accf4e/crunch-core/src/main/java/org/apache/crunch/materialize/MaterializableMap.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/materialize/MaterializableMap.java b/crunch-core/src/main/java/org/apache/crunch/materialize/MaterializableMap.java index 69082e2..d8c98d0 100644 --- a/crunch-core/src/main/java/org/apache/crunch/materialize/MaterializableMap.java +++ b/crunch-core/src/main/java/org/apache/crunch/materialize/MaterializableMap.java @@ -17,6 +17,7 @@ */ package org.apache.crunch.materialize; +import java.io.Serializable; import java.util.AbstractMap; import java.util.HashMap; import java.util.Map; @@ -24,27 +25,47 @@ import java.util.Set; import org.apache.crunch.Pair; -public class MaterializableMap extends AbstractMap { +public class MaterializableMap extends AbstractMap implements Serializable { - private Iterable> iterable; - private Set> entrySet; + private transient Iterable> iterable; + private Map delegate; public MaterializableMap(Iterable> iterable) { this.iterable = iterable; } - private Set> toMapEntries(Iterable> xs) { - HashMap m = new HashMap(); - for (Pair x : xs) - m.put(x.first(), x.second()); - return m.entrySet(); + private Map delegate() { + if (delegate == null) { + delegate = new HashMap(); + for (Pair x : iterable) { + delegate.put(x.first(), x.second()); + } + } + return delegate; } @Override public Set> entrySet() { - if (entrySet == null) - entrySet = toMapEntries(iterable); - return entrySet; + return delegate().entrySet(); } + @Override + public V get(Object key) { + return delegate().get(key); + } + + @Override + public boolean containsKey(Object key) { + return delegate().containsKey(key); + } + + @Override + public int hashCode() { + return delegate().hashCode(); + } + + @Override + public boolean equals(Object other) { + return delegate().equals(other); + } } \ No newline at end of file