crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject crunch git commit: CRUNCH-540: Make AvroReflectDeepCopier serializable (for real though.)
Date Wed, 08 Jul 2015 19:52:35 GMT
Repository: crunch
Updated Branches:
  refs/heads/master 7f85ee581 -> 2bd612eb9


CRUNCH-540: Make AvroReflectDeepCopier serializable (for real though.)


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/2bd612eb
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/2bd612eb
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/2bd612eb

Branch: refs/heads/master
Commit: 2bd612eb9c115f857f5776f58fbd336028a438b3
Parents: 7f85ee5
Author: Josh Wills <jwills@apache.org>
Authored: Tue Jul 7 14:16:07 2015 -0700
Committer: Josh Wills <jwills@apache.org>
Committed: Tue Jul 7 14:16:07 2015 -0700

----------------------------------------------------------------------
 .../crunch/types/avro/AvroDeepCopier.java       |  8 +++---
 .../org/apache/crunch/types/avro/AvroType.java  |  2 +-
 .../crunch/types/avro/AvroDeepCopierTest.java   | 30 ++++++++++++++++++++
 3 files changed, 35 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/2bd612eb/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroDeepCopier.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroDeepCopier.java b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroDeepCopier.java
index dcc4a19..5000f8d 100644
--- a/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroDeepCopier.java
+++ b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroDeepCopier.java
@@ -99,10 +99,10 @@ abstract class AvroDeepCopier<T> implements DeepCopier<T>,
Serializable {
    */
   public static class AvroReflectDeepCopier<T> extends AvroDeepCopier<T> {
 
-    private DatumReader<T> datumReader;
-    private DatumWriter<T> datumWriter;
-    private BinaryEncoder binaryEncoder;
-    private BinaryDecoder binaryDecoder;
+    private transient DatumReader<T> datumReader;
+    private transient DatumWriter<T> datumWriter;
+    private transient BinaryEncoder binaryEncoder;
+    private transient BinaryDecoder binaryDecoder;
     private final Class<T> valueClass;
 
     public AvroReflectDeepCopier(Class<T> valueClass, Schema schema) {

http://git-wip-us.apache.org/repos/asf/crunch/blob/2bd612eb/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroType.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroType.java b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroType.java
index 528a600..97278f0 100644
--- a/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroType.java
+++ b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroType.java
@@ -70,7 +70,7 @@ public class AvroType<T> implements PType<T> {
   private final List<PType> subTypes;
   private AvroRecordType recordType;
   private DeepCopier<T> deepCopier;
-  private boolean initialized = false;
+  private transient boolean initialized = false;
 
   public AvroType(Class<T> typeClass, Schema schema, DeepCopier<T> deepCopier,
PType... ptypes) {
     this(typeClass, schema, IdentityFn.getInstance(), IdentityFn.getInstance(), deepCopier,
null, ptypes);

http://git-wip-us.apache.org/repos/asf/crunch/blob/2bd612eb/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroDeepCopierTest.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroDeepCopierTest.java
b/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroDeepCopierTest.java
index da8dd28..4f5cb75 100644
--- a/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroDeepCopierTest.java
+++ b/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroDeepCopierTest.java
@@ -22,11 +22,16 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotSame;
 import static org.junit.Assert.assertNull;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
 import java.nio.ByteBuffer;
 
 import com.google.common.collect.Lists;
 import org.apache.avro.generic.GenericData.Record;
 import org.apache.crunch.test.Person;
+import org.apache.crunch.types.PType;
 import org.apache.crunch.types.avro.AvroDeepCopier.AvroSpecificDeepCopier;
 import org.apache.hadoop.conf.Configuration;
 import org.junit.Test;
@@ -88,6 +93,31 @@ public class AvroDeepCopierTest {
   }
 
   @Test
+  public void testSerializableReflectPType() throws Exception {
+    ReflectedPerson person = new ReflectedPerson();
+    person.setName("John Doe");
+    person.setAge(42);
+    person.setSiblingnames(Lists.<String>newArrayList());
+
+    PType<ReflectedPerson> rptype = Avros.reflects(ReflectedPerson.class);
+    rptype.initialize(new Configuration());
+    ReflectedPerson copy1 = rptype.getDetachedValue(person);
+    assertEquals(copy1, person);
+
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    ObjectOutputStream oos = new ObjectOutputStream(baos);
+    oos.writeObject(rptype);
+    oos.close();
+
+    ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(baos.toByteArray()));
+    rptype = (PType<ReflectedPerson>) ois.readObject();
+
+    rptype.initialize(new Configuration());
+    ReflectedPerson copy2 = rptype.getDetachedValue(person);
+    assertEquals(person, copy2);
+  }
+
+  @Test
   public void testDeepCopyReflect_Null() {
     AvroDeepCopier<ReflectedPerson> avroDeepCopier = new AvroDeepCopier.AvroReflectDeepCopier<ReflectedPerson>(
         ReflectedPerson.class, Avros.reflects(ReflectedPerson.class).getSchema());


Mime
View raw message