crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject [1/2] git commit: CRUNCH-472: Support Java enum serialization from Scrunch
Date Fri, 03 Oct 2014 03:13:43 GMT
Repository: crunch
Updated Branches:
  refs/heads/master 3740f3b1b -> 1e2ea6eb8


CRUNCH-472: Support Java enum serialization from Scrunch


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/f6ed0ca3
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/f6ed0ca3
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/f6ed0ca3

Branch: refs/heads/master
Commit: f6ed0ca3ae532df2a701d65b986eadc27c0b1a2d
Parents: 3740f3b
Author: Josh Wills <jwills@apache.org>
Authored: Tue Sep 30 08:58:28 2014 -0700
Committer: Josh Wills <jwills@apache.org>
Committed: Thu Oct 2 20:04:05 2014 -0700

----------------------------------------------------------------------
 .../src/main/java/org/apache/crunch/types/PTypes.java        | 3 ++-
 .../main/scala/org/apache/crunch/scrunch/Conversions.scala   | 4 ++++
 .../main/scala/org/apache/crunch/scrunch/PTypeFamily.scala   | 6 +++++-
 .../test/scala/org/apache/crunch/scrunch/TupleNTest.scala    | 8 ++++++++
 4 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/f6ed0ca3/crunch-core/src/main/java/org/apache/crunch/types/PTypes.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/types/PTypes.java b/crunch-core/src/main/java/org/apache/crunch/types/PTypes.java
index a5659ea..82604ac 100644
--- a/crunch-core/src/main/java/org/apache/crunch/types/PTypes.java
+++ b/crunch-core/src/main/java/org/apache/crunch/types/PTypes.java
@@ -97,7 +97,8 @@ public class PTypes {
    * Constructs a PType for a Java {@code Enum} type.
    */
   public static <T extends Enum> PType<T> enums(Class<T> type, PTypeFamily
typeFamily) {
-    return typeFamily.derivedImmutable(type, new EnumInputMapper<T>(type), new EnumOutputMapper<T>(),
typeFamily.strings());
+    return typeFamily.derivedImmutable(type, new EnumInputMapper<T>(type), new EnumOutputMapper<T>(),
+        typeFamily.strings());
   }
 
   public static final MapFn<ByteBuffer, BigInteger> BYTE_TO_BIGINT = new MapFn<ByteBuffer,
BigInteger>() {

http://git-wip-us.apache.org/repos/asf/crunch/blob/f6ed0ca3/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Conversions.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Conversions.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Conversions.scala
index 4dec8cf..c7258ee 100644
--- a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Conversions.scala
+++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Conversions.scala
@@ -118,6 +118,10 @@ object PTypeH extends GeneratedTupleConversions with LowPriorityPTypeH
{
   implicit val strings = new PTypeH[String] { def get(ptf: PTypeFamily) = ptf.strings }
   implicit val bytes = new PTypeH[ByteBuffer] { def get(ptf: PTypeFamily) = ptf.bytes }
 
+  implicit def jenums[E <: java.lang.Enum[E] : ClassTag] = new PTypeH[E] {
+    def get(ptf: PTypeFamily): PType[E] = ptf.jenums(implicitly[ClassTag[E]])
+  }
+
   implicit def writables[W <: Writable : ClassTag] = new PTypeH[W] {
     def get(ptf: PTypeFamily): PType[W] = ptf.writables(implicitly[ClassTag[W]])
   }

http://git-wip-us.apache.org/repos/asf/crunch/blob/f6ed0ca3/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala
index 1157a34..0e31299 100644
--- a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala
+++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala
@@ -18,7 +18,7 @@
 package org.apache.crunch.scrunch
 
 import org.apache.crunch.{Pair => CPair, Tuple3 => CTuple3, Tuple4 => CTuple4, TupleN,
Union, MapFn}
-import org.apache.crunch.types.{PType, PTypeFamily => PTF}
+import org.apache.crunch.types.{PType, PTypeFamily => PTF, PTypes}
 import org.apache.crunch.types.writable.{WritableTypeFamily, Writables => CWritables}
 import org.apache.crunch.types.avro.{AvroType, AvroTypeFamily, Avros => CAvros}
 import java.lang.{Long => JLong, Double => JDouble, Integer => JInt, Float =>
JFloat, Boolean => JBoolean}
@@ -140,6 +140,10 @@ trait PTypeFamily extends GeneratedTuplePTypeFamily {
     derivedImmutable(classOf[Boolean], in, out, ptf.booleans())
   }
 
+  def jenums[E <: java.lang.Enum[E] : ClassTag]: PType[E] = {
+    PTypes.enums(implicitly[ClassTag[E]].runtimeClass.asInstanceOf[Class[E]], ptf).asInstanceOf[PType[E]]
+  }
+
   def options[T](ptype: PType[T]) = {
     val in: Union => Option[T] = (x: Union) => { if (x.getIndex() == 0) None else Some(x.getValue.asInstanceOf[T])
}
     val out = (x: Option[T]) => { if (x.isEmpty) new Union(0, null) else new Union(1,
x.get) }

http://git-wip-us.apache.org/repos/asf/crunch/blob/f6ed0ca3/crunch-scrunch/src/test/scala/org/apache/crunch/scrunch/TupleNTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/test/scala/org/apache/crunch/scrunch/TupleNTest.scala b/crunch-scrunch/src/test/scala/org/apache/crunch/scrunch/TupleNTest.scala
index 0810aa3..50e92ed 100644
--- a/crunch-scrunch/src/test/scala/org/apache/crunch/scrunch/TupleNTest.scala
+++ b/crunch-scrunch/src/test/scala/org/apache/crunch/scrunch/TupleNTest.scala
@@ -20,6 +20,8 @@
 
 package org.apache.crunch.scrunch
 
+import org.apache.crunch.types.avro.AvroMode
+
 import org.scalatest.junit.JUnitSuite
 import org.junit.Test
 
@@ -35,6 +37,12 @@ class TupleNTest extends JUnitSuite{
     org.junit.Assert.assertEquals(List(("a", 9)), res.toList)
   }
 
+  @Test def testJavaEnums {
+    val pc = Mem.collectionOf((1, AvroMode.GENERIC), (2, AvroMode.SPECIFIC), (3, AvroMode.REFLECT))
+    val res = pc.map(x => (x._2, x._1)).filter((k, v) => k == AvroMode.SPECIFIC).materialize
+    org.junit.Assert.assertEquals(List((AvroMode.SPECIFIC, 2)), res.toList)
+  }
+
   /**
    * Basically, we just want to validate that we can generate schemas for these classes successfully
    */


Mime
View raw message