Return-Path: X-Original-To: apmail-crunch-commits-archive@www.apache.org Delivered-To: apmail-crunch-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 603951192C for ; Mon, 11 Aug 2014 22:13:10 +0000 (UTC) Received: (qmail 4944 invoked by uid 500); 11 Aug 2014 22:13:10 -0000 Delivered-To: apmail-crunch-commits-archive@crunch.apache.org Received: (qmail 4905 invoked by uid 500); 11 Aug 2014 22:13:10 -0000 Mailing-List: contact commits-help@crunch.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@crunch.apache.org Delivered-To: mailing list commits@crunch.apache.org Received: (qmail 4894 invoked by uid 99); 11 Aug 2014 22:13:10 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 11 Aug 2014 22:13:10 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id D7824814651; Mon, 11 Aug 2014 22:13:09 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jwills@apache.org To: commits@crunch.apache.org Message-Id: <3a07203290b84923a1db3aa6185c18df@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: git commit: CRUNCH-462: MemPipeline should verify that DoFns are Serializable Date: Mon, 11 Aug 2014 22:13:09 +0000 (UTC) Repository: crunch Updated Branches: refs/heads/apache-crunch-0.8 dd4a75675 -> ee7838408 CRUNCH-462: MemPipeline should verify that DoFns are Serializable Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/ee783840 Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/ee783840 Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/ee783840 Branch: refs/heads/apache-crunch-0.8 Commit: ee7838408e032904a157d0f34e10a0eb189db14f Parents: dd4a756 Author: Josh Wills Authored: Mon Aug 11 11:41:02 2014 -0700 Committer: Josh Wills Committed: Mon Aug 11 15:08:00 2014 -0700 ---------------------------------------------------------------------- .../apache/crunch/impl/mem/collect/MemCollection.java | 13 +++++++++++++ .../java/org/apache/crunch/impl/mem/CountersTest.java | 3 ++- 2 files changed, 15 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/ee783840/crunch-core/src/main/java/org/apache/crunch/impl/mem/collect/MemCollection.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/mem/collect/MemCollection.java b/crunch-core/src/main/java/org/apache/crunch/impl/mem/collect/MemCollection.java index becee88..eaaab59 100644 --- a/crunch-core/src/main/java/org/apache/crunch/impl/mem/collect/MemCollection.java +++ b/crunch-core/src/main/java/org/apache/crunch/impl/mem/collect/MemCollection.java @@ -25,6 +25,8 @@ import javassist.util.proxy.MethodFilter; import javassist.util.proxy.MethodHandler; import javassist.util.proxy.ProxyFactory; +import org.apache.commons.lang.SerializationException; +import org.apache.commons.lang.SerializationUtils; import org.apache.crunch.Aggregator; import org.apache.crunch.CachingOptions; import org.apache.crunch.DoFn; @@ -101,6 +103,16 @@ public class MemCollection implements PCollection { return new MemCollection(output, collections[0].getPType()); } + private DoFn verifySerializable(String name, DoFn doFn) { + try { + return (DoFn) SerializationUtils.deserialize(SerializationUtils.serialize(doFn)); + } catch (SerializationException e) { + throw new IllegalStateException( + doFn.getClass().getSimpleName() + " named '" + name + "' cannot be serialized", + e); + } + } + @Override public PCollection parallelDo(DoFn doFn, PType type) { return parallelDo(null, doFn, type); @@ -114,6 +126,7 @@ public class MemCollection implements PCollection { @Override public PCollection parallelDo(String name, DoFn doFn, PType type, ParallelDoOptions options) { + doFn = verifySerializable(name, doFn); InMemoryEmitter emitter = new InMemoryEmitter(); Configuration conf = getPipeline().getConfiguration(); doFn.configure(conf); http://git-wip-us.apache.org/repos/asf/crunch/blob/ee783840/crunch-core/src/test/java/org/apache/crunch/impl/mem/CountersTest.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/test/java/org/apache/crunch/impl/mem/CountersTest.java b/crunch-core/src/test/java/org/apache/crunch/impl/mem/CountersTest.java index 6b3d0fd..78acb2c 100644 --- a/crunch-core/src/test/java/org/apache/crunch/impl/mem/CountersTest.java +++ b/crunch-core/src/test/java/org/apache/crunch/impl/mem/CountersTest.java @@ -24,9 +24,10 @@ import org.apache.crunch.impl.mem.collect.MemCollection; import org.apache.crunch.types.writable.Writables; import org.junit.Test; +import java.io.Serializable; import java.util.Arrays; -public class CountersTest { +public class CountersTest implements Serializable { @Test public void counterTest() throws Exception {