crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gr...@apache.org
Subject git commit: CRUNCH-346 Don't deep copy immutable writable
Date Sat, 22 Feb 2014 09:05:18 GMT
Repository: crunch
Updated Branches:
  refs/heads/master 8a61e3a62 -> 1b8b15315


CRUNCH-346 Don't deep copy immutable writable

Skip deep copying Writables that are primitive, strings, or
immutable derived types.

Also clean up some other issues with deep copying that was not
being done correctly on ByteBuffer and mutable derived types.


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/1b8b1531
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/1b8b1531
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/1b8b1531

Branch: refs/heads/master
Commit: 1b8b153154e226687b33336bd700f468c62dd4d7
Parents: 8a61e3a
Author: Gabriel Reid <greid@apache.org>
Authored: Sun Feb 16 22:40:30 2014 +0100
Committer: Gabriel Reid <greid@apache.org>
Committed: Sat Feb 22 09:29:27 2014 +0100

----------------------------------------------------------------------
 .../org/apache/crunch/types/DeepCopier.java     | 14 ----
 .../org/apache/crunch/types/NoOpDeepCopier.java | 48 +++++++++++
 .../org/apache/crunch/types/PTypeFamily.java    |  6 ++
 .../java/org/apache/crunch/types/PTypes.java    | 10 +--
 .../types/avro/AvroDerivedValueDeepCopier.java  | 49 ++++++++++++
 .../crunch/types/avro/AvroTypeFamily.java       |  5 ++
 .../org/apache/crunch/types/avro/Avros.java     | 27 ++++---
 .../crunch/types/writable/WritableType.java     | 28 ++++++-
 .../types/writable/WritableTypeFamily.java      |  5 ++
 .../apache/crunch/types/writable/Writables.java | 47 ++++++-----
 .../org/apache/crunch/types/PTypeUtilsTest.java |  2 +-
 .../apache/crunch/types/avro/AvroTypeTest.java  | 50 +++++++++++-
 .../org/apache/crunch/types/avro/AvrosTest.java |  9 +--
 .../crunch/types/writable/WritableTypeTest.java | 84 +++++++++++++++++++-
 .../crunch/types/writable/WritablesTest.java    |  3 +-
 15 files changed, 318 insertions(+), 69 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/1b8b1531/crunch-core/src/main/java/org/apache/crunch/types/DeepCopier.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/types/DeepCopier.java b/crunch-core/src/main/java/org/apache/crunch/types/DeepCopier.java
index f146e86..95dad20 100644
--- a/crunch-core/src/main/java/org/apache/crunch/types/DeepCopier.java
+++ b/crunch-core/src/main/java/org/apache/crunch/types/DeepCopier.java
@@ -43,18 +43,4 @@ public interface DeepCopier<T> extends Serializable {
    */
   T deepCopy(T source);
 
-  static class NoOpDeepCopier<V> implements DeepCopier<V> {
-
-    @Override
-    public V deepCopy(V source) {
-      return source;
-    }
-
-    @Override
-    public void initialize(Configuration conf) {
-      // No initialization needed
-    }
-
-  }
-
 }

http://git-wip-us.apache.org/repos/asf/crunch/blob/1b8b1531/crunch-core/src/main/java/org/apache/crunch/types/NoOpDeepCopier.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/types/NoOpDeepCopier.java b/crunch-core/src/main/java/org/apache/crunch/types/NoOpDeepCopier.java
new file mode 100644
index 0000000..d1dfe4f
--- /dev/null
+++ b/crunch-core/src/main/java/org/apache/crunch/types/NoOpDeepCopier.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.crunch.types;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * A {@code DeepCopier} that does nothing, and just returns the input value without copying
anything.
+ */
+public class NoOpDeepCopier<T> implements DeepCopier<T> {
+
+  private NoOpDeepCopier() {}
+
+  /**
+   * Static factory method.
+   */
+  public static <T> NoOpDeepCopier<T> create() {
+    return new NoOpDeepCopier<T>();
+  }
+
+
+  @Override
+  public T deepCopy(T source) {
+    return source;
+  }
+
+  @Override
+  public void initialize(Configuration conf) {
+    // No initialization needed
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/crunch/blob/1b8b1531/crunch-core/src/main/java/org/apache/crunch/types/PTypeFamily.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/types/PTypeFamily.java b/crunch-core/src/main/java/org/apache/crunch/types/PTypeFamily.java
index 0ad324a..bd38b87 100644
--- a/crunch-core/src/main/java/org/apache/crunch/types/PTypeFamily.java
+++ b/crunch-core/src/main/java/org/apache/crunch/types/PTypeFamily.java
@@ -69,6 +69,12 @@ public interface PTypeFamily {
 
   <S, T> PType<T> derived(Class<T> clazz, MapFn<S, T> inputFn, MapFn<T,
S> outputFn, PType<S> base);
 
+  /**
+   * A derived type whose values are immutable. This variaion of derived exists to optimize
for the case
+   * where deep-copying of data is never needed.
+   */
+  <S, T> PType<T> derivedImmutable(Class<T> clazz, MapFn<S, T> inputFn,
MapFn<T, S> outputFn, PType<S> base);
+
   PType<Union> unionOf(PType<?>... ptypes);
 
   <K, V> PTableType<K, V> tableOf(PType<K> key, PType<V> value);

http://git-wip-us.apache.org/repos/asf/crunch/blob/1b8b1531/crunch-core/src/main/java/org/apache/crunch/types/PTypes.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/types/PTypes.java b/crunch-core/src/main/java/org/apache/crunch/types/PTypes.java
index e701747..a5659ea 100644
--- a/crunch-core/src/main/java/org/apache/crunch/types/PTypes.java
+++ b/crunch-core/src/main/java/org/apache/crunch/types/PTypes.java
@@ -47,14 +47,14 @@ public class PTypes {
    * A PType for Java's {@link BigInteger} type.
    */
   public static PType<BigInteger> bigInt(PTypeFamily typeFamily) {
-    return typeFamily.derived(BigInteger.class, BYTE_TO_BIGINT, BIGINT_TO_BYTE, typeFamily.bytes());
+    return typeFamily.derivedImmutable(BigInteger.class, BYTE_TO_BIGINT, BIGINT_TO_BYTE,
typeFamily.bytes());
   }
 
   /**
    * A PType for Java's {@link UUID} type.
    */
   public static PType<UUID> uuid(PTypeFamily ptf) {
-    return ptf.derived(UUID.class, BYTE_TO_UUID, UUID_TO_BYTE, ptf.bytes());
+    return ptf.derivedImmutable(UUID.class, BYTE_TO_UUID, UUID_TO_BYTE, ptf.bytes());
   }
 
   /**
@@ -69,7 +69,7 @@ public class PTypes {
    * Constructs a PType for the given protocol buffer.
    */
   public static <T extends Message> PType<T> protos(Class<T> clazz, PTypeFamily
typeFamily) {
-    return typeFamily.derived(clazz, new ProtoInputMapFn<T>(clazz), new ProtoOutputMapFn<T>(),
typeFamily.bytes());
+    return typeFamily.derivedImmutable(clazz, new ProtoInputMapFn<T>(clazz), new ProtoOutputMapFn<T>(),
typeFamily.bytes());
   }
 
   /**
@@ -80,7 +80,7 @@ public class PTypes {
       Class<T> clazz,
       PTypeFamily typeFamily,
       SerializableSupplier<ExtensionRegistry> supplier) {
-    return typeFamily.derived(clazz,
+    return typeFamily.derivedImmutable(clazz,
         new ProtoInputMapFn<T>(clazz, supplier),
         new ProtoOutputMapFn<T>(),
         typeFamily.bytes());
@@ -97,7 +97,7 @@ public class PTypes {
    * Constructs a PType for a Java {@code Enum} type.
    */
   public static <T extends Enum> PType<T> enums(Class<T> type, PTypeFamily
typeFamily) {
-    return typeFamily.derived(type, new EnumInputMapper<T>(type), new EnumOutputMapper<T>(),
typeFamily.strings());
+    return typeFamily.derivedImmutable(type, new EnumInputMapper<T>(type), new EnumOutputMapper<T>(),
typeFamily.strings());
   }
 
   public static final MapFn<ByteBuffer, BigInteger> BYTE_TO_BIGINT = new MapFn<ByteBuffer,
BigInteger>() {

http://git-wip-us.apache.org/repos/asf/crunch/blob/1b8b1531/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroDerivedValueDeepCopier.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroDerivedValueDeepCopier.java
b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroDerivedValueDeepCopier.java
new file mode 100644
index 0000000..e872dd2
--- /dev/null
+++ b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroDerivedValueDeepCopier.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.crunch.types.avro;
+
+import org.apache.crunch.MapFn;
+import org.apache.crunch.types.DeepCopier;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * A DeepCopier specific to Avro derived types.
+ */
+public class AvroDerivedValueDeepCopier<T, S> implements DeepCopier {
+
+  private final MapFn<T,S> derivedToAvroFn;
+  private final MapFn<S,T> avroToDerivedFn;
+  private final AvroType<S> avroBaseType;
+
+  public AvroDerivedValueDeepCopier(MapFn<T,S> derivedToAvroFn, MapFn<S,T> avroToDerivedFn,
AvroType<S> avroBaseType) {
+    this.derivedToAvroFn = derivedToAvroFn;
+    this.avroToDerivedFn = avroToDerivedFn;
+    this.avroBaseType = avroBaseType;
+  }
+
+  @Override
+  public void initialize(Configuration conf) {
+    avroBaseType.initialize(conf);
+  }
+
+  @Override
+  public Object deepCopy(Object source) {
+    return avroToDerivedFn.map(avroBaseType.getDetachedValue(derivedToAvroFn.map((T) source)));
+  }
+}

http://git-wip-us.apache.org/repos/asf/crunch/blob/1b8b1531/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroTypeFamily.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroTypeFamily.java b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroTypeFamily.java
index ba8add6..c70b894 100644
--- a/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroTypeFamily.java
+++ b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroTypeFamily.java
@@ -164,6 +164,11 @@ public class AvroTypeFamily implements PTypeFamily {
   }
 
   @Override
+  public <S, T> PType<T> derivedImmutable(Class<T> clazz, MapFn<S, T>
inputFn, MapFn<T, S> outputFn, PType<S> base) {
+    return Avros.derivedImmutable(clazz, inputFn, outputFn, base);
+  }
+
+  @Override
   public PType<Union> unionOf(PType<?>... ptypes) {
     return Avros.unionOf(ptypes);
   }

http://git-wip-us.apache.org/repos/asf/crunch/blob/1b8b1531/crunch-core/src/main/java/org/apache/crunch/types/avro/Avros.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/types/avro/Avros.java b/crunch-core/src/main/java/org/apache/crunch/types/avro/Avros.java
index 8f1dae0..a051a5f 100644
--- a/crunch-core/src/main/java/org/apache/crunch/types/avro/Avros.java
+++ b/crunch-core/src/main/java/org/apache/crunch/types/avro/Avros.java
@@ -30,6 +30,10 @@ import java.util.List;
 import java.util.Map;
 
 import com.google.common.base.Charsets;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import org.apache.avro.Schema;
 import org.apache.avro.Schema.Type;
 import org.apache.avro.generic.GenericData;
@@ -52,8 +56,8 @@ import org.apache.crunch.Union;
 import org.apache.crunch.fn.CompositeMapFn;
 import org.apache.crunch.fn.IdentityFn;
 import org.apache.crunch.types.CollectionDeepCopier;
-import org.apache.crunch.types.DeepCopier;
 import org.apache.crunch.types.MapDeepCopier;
+import org.apache.crunch.types.NoOpDeepCopier;
 import org.apache.crunch.types.PTableType;
 import org.apache.crunch.types.PType;
 import org.apache.crunch.types.PTypes;
@@ -66,11 +70,6 @@ import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.mapreduce.TaskInputOutputContext;
 import org.apache.hadoop.util.ReflectionUtils;
 
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
 /**
  * Defines static methods that are analogous to the methods defined in
  * {@link AvroTypeFamily} for convenient static importing.
@@ -182,7 +181,7 @@ public class Avros {
   };
 
   private static final AvroType<String> strings = new AvroType<String>(String.class,
Schema.create(Schema.Type.STRING),
-      UTF8_TO_STRING, STRING_TO_UTF8, new DeepCopier.NoOpDeepCopier<String>(), AvroType.AvroRecordType.GENERIC);
+      UTF8_TO_STRING, STRING_TO_UTF8, NoOpDeepCopier.<String>create(), AvroType.AvroRecordType.GENERIC);
   private static final AvroType<Void> nulls = create(Void.class, Schema.Type.NULL);
   private static final AvroType<Long> longs = create(Long.class, Schema.Type.LONG);
   private static final AvroType<Integer> ints = create(Integer.class, Schema.Type.INT);
@@ -191,7 +190,7 @@ public class Avros {
   private static final AvroType<Boolean> booleans = create(Boolean.class, Schema.Type.BOOLEAN);
   private static final AvroType<ByteBuffer> bytes = new AvroType<ByteBuffer>(ByteBuffer.class,
       Schema.create(Schema.Type.BYTES), BYTES_IN, IdentityFn.getInstance(),
-      new DeepCopier.NoOpDeepCopier<ByteBuffer>(), AvroType.AvroRecordType.GENERIC);
+      NoOpDeepCopier.<ByteBuffer>create(), AvroType.AvroRecordType.GENERIC);
 
   private static final Map<Class<?>, PType<?>> PRIMITIVES = ImmutableMap.<Class<?>,
PType<?>> builder()
       .put(String.class, strings).put(Long.class, longs).put(Integer.class, ints).put(Float.class,
floats)
@@ -215,7 +214,7 @@ public class Avros {
   }
 
   private static <T> AvroType<T> create(Class<T> clazz, Schema.Type schemaType)
{
-    return new AvroType<T>(clazz, Schema.create(schemaType), new DeepCopier.NoOpDeepCopier<T>());
+    return new AvroType<T>(clazz, Schema.create(schemaType), NoOpDeepCopier.<T>create());
   }
 
   public static final AvroType<Void> nulls() {
@@ -812,7 +811,15 @@ public class Avros {
       PType<S> base) {
     AvroType<S> abase = (AvroType<S>) base;
     return new AvroType<T>(clazz, abase.getSchema(), new CompositeMapFn(abase.getInputMapFn(),
inputFn),
-        new CompositeMapFn(outputFn, abase.getOutputMapFn()), new DeepCopier.NoOpDeepCopier<T>(),
abase.getRecordType(),
+        new CompositeMapFn(outputFn, abase.getOutputMapFn()), new AvroDerivedValueDeepCopier(outputFn,
inputFn, abase),
+        abase.getRecordType(), base.getSubTypes().toArray(new PType[0]));
+  }
+
+  public static final <S, T> AvroType<T> derivedImmutable(Class<T> clazz,
MapFn<S, T> inputFn, MapFn<T, S> outputFn,
+                                                 PType<S> base) {
+    AvroType<S> abase = (AvroType<S>) base;
+    return new AvroType<T>(clazz, abase.getSchema(), new CompositeMapFn(abase.getInputMapFn(),
inputFn),
+        new CompositeMapFn(outputFn, abase.getOutputMapFn()), NoOpDeepCopier.<T>create(),
abase.getRecordType(),
         base.getSubTypes().toArray(new PType[0]));
   }
 

http://git-wip-us.apache.org/repos/asf/crunch/blob/1b8b1531/crunch-core/src/main/java/org/apache/crunch/types/writable/WritableType.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/types/writable/WritableType.java
b/crunch-core/src/main/java/org/apache/crunch/types/writable/WritableType.java
index 10cd24d..ea3019b 100644
--- a/crunch-core/src/main/java/org/apache/crunch/types/writable/WritableType.java
+++ b/crunch-core/src/main/java/org/apache/crunch/types/writable/WritableType.java
@@ -19,6 +19,7 @@ package org.apache.crunch.types.writable;
 
 import java.util.List;
 
+import com.google.common.collect.ImmutableList;
 import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.crunch.MapFn;
 import org.apache.crunch.io.ReadableSourceTarget;
@@ -31,8 +32,6 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Writable;
 
-import com.google.common.collect.ImmutableList;
-
 public class WritableType<T, W extends Writable> implements PType<T> {
 
   private final Class<T> typeClass;
@@ -44,14 +43,32 @@ public class WritableType<T, W extends Writable> implements PType<T>
{
   private final List<PType> subTypes;
   private boolean initialized = false;
 
+  /**
+   * Factory method for a new WritableType instance whose type class is immutable.
+   * <p/>
+   * No checking is done to ensure that instances of the type class are immutable, but deep
copying will be skipped
+   * for instances denoted by the created PType.
+   */
+  public static <T, W extends Writable> WritableType<T, W> immutableType(Class<T>
typeClass, Class<W> writableClass,
+                                                                         MapFn<W, T>
inputDoFn, MapFn<T, W> outputDoFn,
+                                                                         PType... subTypes)
{
+    return new WritableType<T, W>(typeClass, writableClass, inputDoFn, outputDoFn,
+                                  null, subTypes);
+  }
+
   public WritableType(Class<T> typeClass, Class<W> writableClass, MapFn<W,
T> inputDoFn,
-      MapFn<T, W> outputDoFn, PType... subTypes) {
+                       MapFn<T, W> outputDoFn, PType... subTypes) {
+    this(typeClass, writableClass, inputDoFn, outputDoFn, new WritableDeepCopier<W>(writableClass),
subTypes);
+  }
+
+  private WritableType(Class<T> typeClass, Class<W> writableClass, MapFn<W,
T> inputDoFn,
+      MapFn<T, W> outputDoFn, DeepCopier<W> deepCopier, PType... subTypes) {
     this.typeClass = typeClass;
     this.writableClass = writableClass;
     this.inputFn = inputDoFn;
     this.outputFn = outputDoFn;
     this.converter = new WritableValueConverter(writableClass);
-    this.deepCopier = new WritableDeepCopier<W>(writableClass);
+    this.deepCopier = deepCopier;
     this.subTypes = ImmutableList.<PType> builder().add(subTypes).build();
   }
 
@@ -118,6 +135,9 @@ public class WritableType<T, W extends Writable> implements PType<T>
{
 
   @Override
   public T getDetachedValue(T value) {
+    if (deepCopier == null) {
+      return value;
+    }
     if (!initialized) {
       throw new IllegalStateException("Cannot call getDetachedValue on an uninitialized PType");
     }

http://git-wip-us.apache.org/repos/asf/crunch/blob/1b8b1531/crunch-core/src/main/java/org/apache/crunch/types/writable/WritableTypeFamily.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/types/writable/WritableTypeFamily.java
b/crunch-core/src/main/java/org/apache/crunch/types/writable/WritableTypeFamily.java
index 5754b4d..2f100a0 100644
--- a/crunch-core/src/main/java/org/apache/crunch/types/writable/WritableTypeFamily.java
+++ b/crunch-core/src/main/java/org/apache/crunch/types/writable/WritableTypeFamily.java
@@ -147,6 +147,11 @@ public class WritableTypeFamily implements PTypeFamily {
   }
 
   @Override
+  public <S, T> PType<T> derivedImmutable(Class<T> clazz, MapFn<S, T>
inputFn, MapFn<T, S> outputFn, PType<S> base) {
+    return Writables.derivedImmutable(clazz, inputFn, outputFn, base);
+  }
+
+  @Override
   public PType<Union> unionOf(PType<?>... ptypes) {
     return Writables.unionOf(ptypes);
   }

http://git-wip-us.apache.org/repos/asf/crunch/blob/1b8b1531/crunch-core/src/main/java/org/apache/crunch/types/writable/Writables.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/types/writable/Writables.java b/crunch-core/src/main/java/org/apache/crunch/types/writable/Writables.java
index a121ae3..d087ca3 100644
--- a/crunch-core/src/main/java/org/apache/crunch/types/writable/Writables.java
+++ b/crunch-core/src/main/java/org/apache/crunch/types/writable/Writables.java
@@ -262,27 +262,23 @@ public class Writables {
     }
   };
 
-  private static <S, W extends Writable> WritableType<S, W> create(Class<S>
typeClass, Class<W> writableClass,
-      MapFn<W, S> inputDoFn, MapFn<S, W> outputDoFn) {
-    return new WritableType<S, W>(typeClass, writableClass, inputDoFn, outputDoFn);
-  }
-
-  private static final WritableType<Void, NullWritable> nulls = create(Void.class,
NullWritable.class,
-      NULL_WRITABLE_TO_VOID, VOID_TO_NULL_WRITABLE);
-  private static final WritableType<String, Text> strings = create(String.class, Text.class,
TEXT_TO_STRING,
-      STRING_TO_TEXT);
-  private static final WritableType<Long, LongWritable> longs = create(Long.class,
LongWritable.class, LW_TO_LONG,
-      LONG_TO_LW);
-  private static final WritableType<Integer, IntWritable> ints = create(Integer.class,
IntWritable.class, IW_TO_INT,
-      INT_TO_IW);
-  private static final WritableType<Float, FloatWritable> floats = create(Float.class,
FloatWritable.class,
-      FW_TO_FLOAT, FLOAT_TO_FW);
-  private static final WritableType<Double, DoubleWritable> doubles = create(Double.class,
DoubleWritable.class,
-      DW_TO_DOUBLE, DOUBLE_TO_DW);
-  private static final WritableType<Boolean, BooleanWritable> booleans = create(Boolean.class,
BooleanWritable.class,
-      BW_TO_BOOLEAN, BOOLEAN_TO_BW);
-  private static final WritableType<ByteBuffer, BytesWritable> bytes = create(ByteBuffer.class,
BytesWritable.class,
-      BW_TO_BB, BB_TO_BW);
+
+  private static final WritableType<Void, NullWritable> nulls = WritableType.immutableType(
+      Void.class, NullWritable.class, NULL_WRITABLE_TO_VOID, VOID_TO_NULL_WRITABLE);
+  private static final WritableType<String, Text> strings = WritableType.immutableType(
+      String.class, Text.class, TEXT_TO_STRING, STRING_TO_TEXT);
+  private static final WritableType<Long, LongWritable> longs = WritableType.immutableType(
+      Long.class, LongWritable.class, LW_TO_LONG, LONG_TO_LW);
+  private static final WritableType<Integer, IntWritable> ints = WritableType.immutableType(
+      Integer.class, IntWritable.class, IW_TO_INT, INT_TO_IW);
+  private static final WritableType<Float, FloatWritable> floats = WritableType.immutableType(
+      Float.class, FloatWritable.class, FW_TO_FLOAT, FLOAT_TO_FW);
+  private static final WritableType<Double, DoubleWritable> doubles = WritableType.immutableType(
+      Double.class, DoubleWritable.class, DW_TO_DOUBLE, DOUBLE_TO_DW);
+  private static final WritableType<Boolean, BooleanWritable> booleans = WritableType.immutableType(
+      Boolean.class, BooleanWritable.class, BW_TO_BOOLEAN, BOOLEAN_TO_BW);
+  private static final WritableType<ByteBuffer, BytesWritable> bytes = new WritableType(
+      ByteBuffer.class, BytesWritable.class, BW_TO_BB, BB_TO_BW);
 
   private static final Map<Class<?>, PType<?>> PRIMITIVES = ImmutableMap.<Class<?>,
PType<?>> builder()
       .put(String.class, strings).put(Long.class, longs).put(Integer.class, ints).put(Float.class,
floats)
@@ -344,7 +340,7 @@ public class Writables {
 
   public static <W extends Writable> WritableType<W, W> writables(Class<W>
clazz) {
     MapFn wIdentity = IdentityFn.getInstance();
-    return new WritableType<W, W>(clazz, clazz, wIdentity, wIdentity);
+    return new WritableType(clazz, clazz, wIdentity, wIdentity);
   }
 
   public static <K, V> WritableTableType<K, V> tableOf(PType<K> key, PType<V>
value) {
@@ -677,6 +673,13 @@ public class Writables {
     return new WritableType(clazz, wt.getSerializationClass(), input, output, base.getSubTypes().toArray(new
PType[0]));
   }
 
+  public static <S, T> PType<T> derivedImmutable(Class<T> clazz, MapFn<S,
T> inputFn, MapFn<T, S> outputFn, PType<S> base) {
+    WritableType<S, ?> wt = (WritableType<S, ?>) base;
+    MapFn input = new CompositeMapFn(wt.getInputMapFn(), inputFn);
+    MapFn output = new CompositeMapFn(outputFn, wt.getOutputMapFn());
+    return WritableType.immutableType(clazz, wt.getSerializationClass(), input, output, base.getSubTypes().toArray(new
PType[0]));
+  }
+
   private static class ArrayCollectionMapFn<T> extends MapFn<GenericArrayWritable,
Collection<T>> {
     private Class<Writable> clazz;
     private final MapFn<Object, T> mapFn;

http://git-wip-us.apache.org/repos/asf/crunch/blob/1b8b1531/crunch-core/src/test/java/org/apache/crunch/types/PTypeUtilsTest.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/test/java/org/apache/crunch/types/PTypeUtilsTest.java b/crunch-core/src/test/java/org/apache/crunch/types/PTypeUtilsTest.java
index e6fd90c..f4ba697 100644
--- a/crunch-core/src/test/java/org/apache/crunch/types/PTypeUtilsTest.java
+++ b/crunch-core/src/test/java/org/apache/crunch/types/PTypeUtilsTest.java
@@ -77,7 +77,7 @@ public class PTypeUtilsTest {
 
   @Test
   public void testAvroRegistered() {
-    AvroType<Utf8> at = new AvroType<Utf8>(Utf8.class, Schema.create(Schema.Type.STRING),
new DeepCopier.NoOpDeepCopier<Utf8>());
+    AvroType<Utf8> at = new AvroType<Utf8>(Utf8.class, Schema.create(Schema.Type.STRING),
NoOpDeepCopier.<Utf8>create());
     Avros.register(Utf8.class, at);
     assertEquals(at, Avros.records(Utf8.class));
   }

http://git-wip-us.apache.org/repos/asf/crunch/blob/1b8b1531/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroTypeTest.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroTypeTest.java b/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroTypeTest.java
index ca9a352..481444f 100644
--- a/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroTypeTest.java
+++ b/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroTypeTest.java
@@ -17,28 +17,32 @@
  */
 package org.apache.crunch.types.avro;
 
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotSame;
 import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
 
+import java.nio.ByteBuffer;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.generic.GenericData.Record;
 import org.apache.crunch.Pair;
 import org.apache.crunch.TupleN;
 import org.apache.crunch.test.Person;
 import org.apache.crunch.test.StringWrapper;
+import org.apache.crunch.types.PType;
+import org.apache.crunch.types.PTypes;
 import org.apache.hadoop.conf.Configuration;
 import org.junit.Test;
 
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
 public class AvroTypeTest {
 
   @Test
@@ -283,4 +287,44 @@ public class AvroTypeTest {
     assertNotSame(person, detachedTuple.get(0));
   }
 
+  @Test
+  public void testGetDetachedValue_ImmutableDerived() {
+    PType<UUID> uuidType = PTypes.uuid(AvroTypeFamily.getInstance());
+    uuidType.initialize(new Configuration());
+
+    UUID uuid = new UUID(1L, 1L);
+    UUID detached = uuidType.getDetachedValue(uuid);
+
+    assertSame(uuid, detached);
+  }
+
+  @Test
+  public void testGetDetachedValue_MutableDerived() {
+    PType<StringWrapper> jsonType = PTypes.jsonString(StringWrapper.class, AvroTypeFamily.getInstance());
+    jsonType.initialize(new Configuration());
+
+    StringWrapper stringWrapper = new StringWrapper();
+    stringWrapper.setValue("test");
+
+    StringWrapper detachedValue = jsonType.getDetachedValue(stringWrapper);
+
+    assertNotSame(stringWrapper, detachedValue);
+    assertEquals(stringWrapper, detachedValue);
+  }
+
+  @Test
+  public void testGetDetachedValue_Bytes() {
+    byte[] buffer = new byte[]{1, 2, 3};
+    AvroType<ByteBuffer> byteType = Avros.bytes();
+    byteType.initialize(new Configuration());
+
+    ByteBuffer detachedValue = byteType.getDetachedValue(ByteBuffer.wrap(buffer));
+
+    byte[] detachedBuffer = new byte[buffer.length];
+    detachedValue.get(detachedBuffer);
+
+    assertArrayEquals(buffer, detachedBuffer);
+    buffer[0] = 99;
+    assertEquals(detachedBuffer[0], 1);
+  }
 }

http://git-wip-us.apache.org/repos/asf/crunch/blob/1b8b1531/crunch-core/src/test/java/org/apache/crunch/types/avro/AvrosTest.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/test/java/org/apache/crunch/types/avro/AvrosTest.java b/crunch-core/src/test/java/org/apache/crunch/types/avro/AvrosTest.java
index 5622a56..bbfcee5 100644
--- a/crunch-core/src/test/java/org/apache/crunch/types/avro/AvrosTest.java
+++ b/crunch-core/src/test/java/org/apache/crunch/types/avro/AvrosTest.java
@@ -27,6 +27,8 @@ import java.nio.ByteBuffer;
 import java.util.Collection;
 import java.util.Collections;
 
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
 import org.apache.avro.Schema;
 import org.apache.avro.Schema.Type;
 import org.apache.avro.generic.GenericData;
@@ -40,7 +42,7 @@ import org.apache.crunch.TupleN;
 import org.apache.crunch.test.CrunchTestSupport;
 import org.apache.crunch.test.Person;
 import org.apache.crunch.test.StringWrapper;
-import org.apache.crunch.types.DeepCopier;
+import org.apache.crunch.types.NoOpDeepCopier;
 import org.apache.crunch.types.PTableType;
 import org.apache.crunch.types.PType;
 import org.apache.hadoop.conf.Configuration;
@@ -49,9 +51,6 @@ import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.mapreduce.TaskInputOutputContext;
 import org.junit.Test;
 
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-
 /**
  * TODO test Avros.register and Avros.containers
  */
@@ -235,7 +234,7 @@ public class AvrosTest {
 
   @Test
   public void testIsPrimitive_TruePrimitiveValue() {
-    AvroType truePrimitiveAvroType = new AvroType(int.class, Schema.create(Type.INT), new
DeepCopier.NoOpDeepCopier());
+    AvroType truePrimitiveAvroType = new AvroType(int.class, Schema.create(Type.INT), NoOpDeepCopier.create());
     assertTrue(Avros.isPrimitive(truePrimitiveAvroType));
   }
 

http://git-wip-us.apache.org/repos/asf/crunch/blob/1b8b1531/crunch-core/src/test/java/org/apache/crunch/types/writable/WritableTypeTest.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/test/java/org/apache/crunch/types/writable/WritableTypeTest.java
b/crunch-core/src/test/java/org/apache/crunch/types/writable/WritableTypeTest.java
index 19a9bfe..8bc0a8e 100644
--- a/crunch-core/src/test/java/org/apache/crunch/types/writable/WritableTypeTest.java
+++ b/crunch-core/src/test/java/org/apache/crunch/types/writable/WritableTypeTest.java
@@ -17,21 +17,29 @@
  */
 package org.apache.crunch.types.writable;
 
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
 
+import java.nio.ByteBuffer;
 import java.util.Collection;
 import java.util.Map;
+import java.util.UUID;
 
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import org.apache.crunch.Pair;
+import org.apache.crunch.test.StringWrapper;
+import org.apache.crunch.types.PType;
+import org.apache.crunch.types.PTypes;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.MapWritable;
 import org.apache.hadoop.io.Text;
 import org.junit.Test;
 
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
 public class WritableTypeTest {
 
   @Test(expected = IllegalStateException.class)
@@ -94,4 +102,74 @@ public class WritableTypeTest {
     assertNotSame(stringTextMap.get("key"), detachedMap.get("key"));
   }
 
+  @Test
+  public void testGetDetachedValue_String() {
+    String s = "test";
+    WritableType<String, Text> stringType = Writables.strings();
+    stringType.initialize(new Configuration());
+    String detached = stringType.getDetachedValue(s);
+
+    assertSame(s, detached);
+  }
+
+
+  @Test
+  public void testGetDetachedValue_Primitive() {
+    WritableType<Integer, IntWritable> intType = Writables.ints();
+    intType.initialize(new Configuration());
+    Integer intValue = Integer.valueOf(42);
+    Integer detachedValue = intType.getDetachedValue(intValue);
+    assertSame(intValue, detachedValue);
+  }
+
+  @Test
+  public void testGetDetachedValue_NonPrimitive() {
+    WritableType<Text, Text> textType = Writables.writables(Text.class);
+    textType.initialize(new Configuration());
+    Text text = new Text("test");
+    Text detachedValue = textType.getDetachedValue(text);
+    assertEquals(text, detachedValue);
+    assertNotSame(text, detachedValue);
+  }
+
+  @Test
+  public void testGetDetachedValue_ImmutableDerived() {
+    PType<UUID> uuidType = PTypes.uuid(WritableTypeFamily.getInstance());
+    uuidType.initialize(new Configuration());
+
+    UUID uuid = new UUID(1L, 1L);
+    UUID detached = uuidType.getDetachedValue(uuid);
+
+    assertSame(uuid, detached);
+  }
+
+  @Test
+  public void testGetDetachedValue_MutableDerived() {
+    PType<StringWrapper> jsonType = PTypes.jsonString(StringWrapper.class, WritableTypeFamily.getInstance());
+    jsonType.initialize(new Configuration());
+
+    StringWrapper stringWrapper = new StringWrapper();
+    stringWrapper.setValue("test");
+
+    StringWrapper detachedValue = jsonType.getDetachedValue(stringWrapper);
+
+    assertNotSame(stringWrapper, detachedValue);
+    assertEquals(stringWrapper, detachedValue);
+  }
+
+  @Test
+  public void testGetDetachedValue_Bytes() {
+    byte[] buffer = new byte[]{1, 2, 3};
+    WritableType<ByteBuffer,BytesWritable> byteType = Writables.bytes();
+    byteType.initialize(new Configuration());
+
+    ByteBuffer detachedValue = byteType.getDetachedValue(ByteBuffer.wrap(buffer));
+
+    byte[] detachedBuffer = new byte[buffer.length];
+    detachedValue.get(detachedBuffer);
+
+    assertArrayEquals(buffer, detachedBuffer);
+    buffer[0] = 99;
+    assertEquals(detachedBuffer[0], 1);
+  }
 }

http://git-wip-us.apache.org/repos/asf/crunch/blob/1b8b1531/crunch-core/src/test/java/org/apache/crunch/types/writable/WritablesTest.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/test/java/org/apache/crunch/types/writable/WritablesTest.java
b/crunch-core/src/test/java/org/apache/crunch/types/writable/WritablesTest.java
index 3a6fc18..70099f2 100644
--- a/crunch-core/src/test/java/org/apache/crunch/types/writable/WritablesTest.java
+++ b/crunch-core/src/test/java/org/apache/crunch/types/writable/WritablesTest.java
@@ -27,6 +27,7 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Collection;
 
+import com.google.common.collect.Lists;
 import org.apache.crunch.Pair;
 import org.apache.crunch.Tuple3;
 import org.apache.crunch.Tuple4;
@@ -45,8 +46,6 @@ import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableUtils;
 import org.junit.Test;
 
-import com.google.common.collect.Lists;
-
 public class WritablesTest {
 
   @Test


Mime
View raw message