crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject git commit: CRUNCH-444: Generate Tuple5..Tuple22 and generalized product Aggregators for Scrunch
Date Thu, 17 Jul 2014 19:22:52 GMT
Repository: crunch
Updated Branches:
  refs/heads/apache-crunch-0.8 00127b9a9 -> 4693cec0f


CRUNCH-444: Generate Tuple5..Tuple22 and generalized product Aggregators for Scrunch


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/4693cec0
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/4693cec0
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/4693cec0

Branch: refs/heads/apache-crunch-0.8
Commit: 4693cec0f58ed577b4e67092b4271d49c6bf18d3
Parents: 00127b9
Author: Josh Wills <jwills@apache.org>
Authored: Tue Jul 15 20:04:26 2014 -0700
Committer: Josh Wills <jwills@apache.org>
Committed: Thu Jul 17 12:15:30 2014 -0700

----------------------------------------------------------------------
 .../src/codegen/make_tuple_conversions.py       |  51 +++++++
 .../src/codegen/make_tuple_ptypefamily.py       |  60 +++++++++
 .../scrunch/AggregatorsIntegrationTest.scala    |  41 ++++++
 .../org/apache/crunch/scrunch/Aggregators.scala |  83 ++++++------
 .../org/apache/crunch/scrunch/Conversions.scala |   2 +-
 .../scrunch/GeneratedTupleConversions.scala     | 129 ++++++++++++++++++
 .../scrunch/GeneratedTuplePTypeFamily.scala     | 135 +++++++++++++++++++
 .../org/apache/crunch/scrunch/PTypeFamily.scala |  20 ++-
 .../org/apache/crunch/scrunch/TupleNTest.scala  |  32 +++++
 9 files changed, 505 insertions(+), 48 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/4693cec0/crunch-scrunch/src/codegen/make_tuple_conversions.py
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/codegen/make_tuple_conversions.py b/crunch-scrunch/src/codegen/make_tuple_conversions.py
new file mode 100755
index 0000000..114ff1e
--- /dev/null
+++ b/crunch-scrunch/src/codegen/make_tuple_conversions.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+print """/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */"""
+print "package org.apache.crunch.scrunch\n"
+print "trait GeneratedTupleConversions {"
+for j in range(5, 23):
+  lets = letters[0: j]
+  types = ", ".join(lets)
+  pth = ", ".join(["%s: PTypeH" % x for x in lets])
+  print "  implicit def tuple%d[%s] = new PTypeH[(%s)] {" % (j, pth, types)
+  print "    def get(ptf: PTypeFamily) = {"
+  implicits = ", ".join(["implicitly[PTypeH[%s]].get(ptf)" % x for x in lets])
+  print "      ptf.tuple%d(%s)" % (j, implicits)
+  print "    }"
+  print "  }\n"
+print "}"

http://git-wip-us.apache.org/repos/asf/crunch/blob/4693cec0/crunch-scrunch/src/codegen/make_tuple_ptypefamily.py
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/codegen/make_tuple_ptypefamily.py b/crunch-scrunch/src/codegen/make_tuple_ptypefamily.py
new file mode 100755
index 0000000..fc7b989
--- /dev/null
+++ b/crunch-scrunch/src/codegen/make_tuple_ptypefamily.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+print """/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */"""
+print "package org.apache.crunch.scrunch\n"
+print "import org.apache.crunch.TupleN"
+print "import org.apache.crunch.types.PType\n"
+
+print "trait GeneratedTuplePTypeFamily extends BasePTypeFamily {"
+print "  import GeneratedTupleHelper._\n"
+for j in range(5, 23):
+  lets = letters[0:j]
+  types = ", ".join(lets)
+  args = ", ".join(["p%d: PType[%s]" % (x, l) for (x, l) in enumerate(lets)])
+  print "  def tuple%d[%s](%s) = {" % (j, types, args)
+
+  inout = ",".join(["t.get(%d).asInstanceOf[%s]" % (x, l) for (x, l) in enumerate(lets)])
+  print "    val in = (t: TupleN) => (%s)" % inout
+
+  outin = ", ".join(lets)
+  outout = ", ".join(["t._%d" % (1 + x) for x in range(j)])
+  print "    val out = (t: (%s)) => tupleN(%s)" % (outin, outout)
+  derout = ", ".join(["p%d" % x for x in range(j)])
+  print "    derived(classOf[(%s)], in, out, ptf.tuples(%s))" % (outin, derout)
+  print "  }\n"
+print "}\n"

http://git-wip-us.apache.org/repos/asf/crunch/blob/4693cec0/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/AggregatorsIntegrationTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/AggregatorsIntegrationTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/AggregatorsIntegrationTest.scala
new file mode 100644
index 0000000..94a6e12
--- /dev/null
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/AggregatorsIntegrationTest.scala
@@ -0,0 +1,41 @@
+/*
+ * *
+ *  * Licensed to the Apache Software Foundation (ASF) under one
+ *  * or more contributor license agreements.  See the NOTICE file
+ *  * distributed with this work for additional information
+ *  * regarding copyright ownership.  The ASF licenses this file
+ *  * to you under the Apache License, Version 2.0 (the
+ *  * "License"); you may not use this file except in compliance
+ *  * with the License.  You may obtain a copy of the License at
+ *  *
+ *  *     http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ *
+ */
+
+package org.apache.crunch.scrunch
+
+import org.junit.Test
+
+class AggregatorsIntegrationTest extends CrunchSuite {
+  @Test def productAggregators {
+    val pipeline = Pipeline.mapReduce[WordCountTest](tempDir.getDefaultConfiguration)
+    val input = tempDir.copyResourceFileName("shakes.txt")
+
+    val fcc = pipeline.read(From.textFile(input))
+      .flatMap(_.toLowerCase.split("\\s+"))
+      .filter(!_.isEmpty())
+      .map(word => (word.slice(0, 1), (1L, word.length)))
+      .groupByKey
+      .combineValues(Aggregators.product[(Long, Int)](Aggregators.sum[Long], Aggregators.max[Int]))
+      .materialize
+    assert(fcc.exists(_ == ("w", (1404, 12))))
+
+    pipeline.done
+  }
+}

http://git-wip-us.apache.org/repos/asf/crunch/blob/4693cec0/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Aggregators.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Aggregators.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Aggregators.scala
index 238a01f..8b0f6b0 100644
--- a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Aggregators.scala
+++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Aggregators.scala
@@ -22,19 +22,19 @@ package org.apache.crunch.scrunch
 
 import org.apache.crunch.fn.{Aggregators => JAgg}
 import org.apache.crunch._
-import com.google.common.collect.ImmutableList
+import com.google.common.collect.{UnmodifiableIterator, ImmutableList}
 import org.apache.hadoop.conf.Configuration
 import java.lang.{Iterable => JIterable}
 import scala.collection.JavaConversions
 import com.twitter.algebird.Monoid
+import scala.reflect.ClassTag
+import java.util
 
 /**
  * Scrunch versions of the common Aggregator types from Crunch.
  */
 object Aggregators {
 
-  import scala.math.Ordering._
-
   def sum[T: Monoid]: Aggregator[T] = sumUsing(implicitly[Monoid[T]])
 
   def sumUsing[T](m: Monoid[T]): Aggregator[T] = new SimpleAggregator[T] {
@@ -176,14 +176,14 @@ object Aggregators {
    * Apply separate aggregators to each component of a {@link Tuple2}.
    */
   def pair[V1, V2](a1: Aggregator[V1], a2: Aggregator[V2]): Aggregator[(V1, V2)] = {
-    return new Aggregators.PairAggregator[V1, V2](a1, a2)
+    new Aggregators.PairAggregator[V1, V2](a1, a2)
   }
 
   /**
    * Apply separate aggregators to each component of a {@link Tuple3}.
    */
   def trip[V1, V2, V3](a1: Aggregator[V1], a2: Aggregator[V2], a3: Aggregator[V3]): Aggregator[(V1, V2, V3)] = {
-    return new Aggregators.TripAggregator[V1, V2, V3](a1, a2, a3)
+    new Aggregators.TripAggregator[V1, V2, V3](a1, a2, a3)
   }
 
   /**
@@ -191,7 +191,15 @@ object Aggregators {
    */
   def quad[V1, V2, V3, V4](a1: Aggregator[V1], a2: Aggregator[V2], a3: Aggregator[V3], a4: Aggregator[V4])
     : Aggregator[(V1, V2, V3, V4)] = {
-    return new Aggregators.QuadAggregator[V1, V2, V3, V4](a1, a2, a3, a4)
+    new Aggregators.QuadAggregator[V1, V2, V3, V4](a1, a2, a3, a4)
+  }
+
+  /**
+   * Apply separate aggregators to each component of a {@link Product} subclass.
+   */
+  def product[T <: Product : ClassTag](aggregators: Aggregator[_]*): Aggregator[T] = {
+    new Aggregators.ProductAggregator[T](Array(aggregators : _*),
+      implicitly[ClassTag[T]].runtimeClass.asInstanceOf[Class[T]])
   }
 
   /**
@@ -202,20 +210,22 @@ object Aggregators {
     }
   }
 
-  private abstract class ProductAggregator[T <: Product](aggs: Array[Aggregator[_]]) extends Aggregator[T] {
+  private class ProductAggregator[T <: Product](val aggs: Array[Aggregator[_]], val clazz: Class[T])
+    extends Aggregator[T] {
+
     def initialize(configuration: Configuration) {
       for (a <- aggs) {
         a.initialize(configuration)
       }
     }
 
-    def reset {
+    def reset() {
       for (a <- aggs) {
-        a.reset
+        a.reset()
       }
     }
 
-    protected def updateTuple(t: T) {
+    override def update(t: T) {
       var i: Int = 0
       while (i < aggs.length) {
         aggs(i).asInstanceOf[Aggregator[Any]].update(t.productElement(i))
@@ -223,49 +233,40 @@ object Aggregators {
       }
     }
 
-    protected def results[T](index: Int): Iterable[T] = {
-      return JavaConversions.iterableAsScalaIterable(aggs(index).results.asInstanceOf[JIterable[T]])
+    override def results: JIterable[T] = {
+      val res = aggs.map(_.results()).map(JavaConversions.iterableAsScalaIterable(_).toList)
+      return new JIterable[T] {
+        override def iterator(): util.Iterator[T] = {
+          return new AggIterator[T](res, clazz)
+        }
+      }
     }
   }
 
-  private class PairAggregator[A, B](val a1: Aggregator[A], val a2: Aggregator[B])
-    extends ProductAggregator[(A, B)](Array(a1, a2)) {
+  private class AggIterator[T](val results: Array[List[_]], val clazz: Class[T]) extends UnmodifiableIterator[T] {
+    var offset = 0
+    val maxoffset = results.map(_.size).min
+    val construct = clazz.getConstructors.find(_.getParameterTypes.length == results.length)
 
-    def update(value: (A, B)) {
-      updateTuple(value)
-    }
+    override def hasNext: Boolean = offset < maxoffset
 
-    def results: JIterable[(A, B)] = {
-      return JavaConversions.asJavaIterable(results[A](0).zip(results[B](1)))
+    override def next(): T = {
+      val refs = results.map(x => x(offset)).asInstanceOf[Array[AnyRef]]
+      offset = offset + 1
+      construct.get.newInstance(refs : _*).asInstanceOf[T]
     }
   }
 
-  private class TripAggregator[A, B, C](val a1: Aggregator[A], val a2: Aggregator[B], val a3: Aggregator[C])
-    extends ProductAggregator[(A, B, C)](Array(a1, a2, a3)) {
-    def update(value: (A, B, C)) {
-      updateTuple(value)
-    }
+  private class PairAggregator[A, B](val a1: Aggregator[A], val a2: Aggregator[B])
+    extends ProductAggregator[(A, B)](Array(a1, a2), classOf[(A, B)]) {
+  }
 
-    def results: JIterable[(A, B, C)] = {
-      return JavaConversions.asJavaIterable(
-        results[A](0).zip(results[B](1)).zip(results[C](2))
-          .map(x => (x._1._1, x._1._2, x._2)))
-    }
+  private class TripAggregator[A, B, C](val a1: Aggregator[A], val a2: Aggregator[B], val a3: Aggregator[C])
+    extends ProductAggregator[(A, B, C)](Array(a1, a2, a3), classOf[(A, B, C)]) {
   }
 
   private class QuadAggregator[A, B, C, D](val a1: Aggregator[A], val a2: Aggregator[B],
                                            val a3: Aggregator[C], val a4: Aggregator[D])
-    extends ProductAggregator[(A, B, C, D)](Array(a1, a2, a3, a4)) {
-
-    def update(value: (A, B, C, D)) {
-      updateTuple(value)
-    }
-
-    def results: JIterable[(A, B, C, D)] = {
-      return JavaConversions.asJavaIterable(
-        (results[A](0).zip(results[B](1))).zip(results[C](2).zip(results[D](3)))
-          .map(x => (x._1._1, x._1._2, x._2._1, x._2._2)))
-    }
+    extends ProductAggregator[(A, B, C, D)](Array(a1, a2, a3, a4), classOf[(A, B, C, D)]) {
   }
-
 }

http://git-wip-us.apache.org/repos/asf/crunch/blob/4693cec0/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Conversions.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Conversions.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Conversions.scala
index 05baebf..ffd85c6 100644
--- a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Conversions.scala
+++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Conversions.scala
@@ -92,7 +92,7 @@ trait LowPriorityPTypeH {
   }
 }
 
-object PTypeH extends LowPriorityPTypeH {
+object PTypeH extends GeneratedTupleConversions with LowPriorityPTypeH {
 
   implicit val longs = new PTypeH[Long] { def get(ptf: PTypeFamily) = ptf.longs }
   implicit val jlongs = new PTypeH[java.lang.Long] { def get(ptf: PTypeFamily) = ptf.jlongs }

http://git-wip-us.apache.org/repos/asf/crunch/blob/4693cec0/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTupleConversions.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTupleConversions.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTupleConversions.scala
new file mode 100644
index 0000000..e2218c5
--- /dev/null
+++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTupleConversions.scala
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch.scrunch
+
+trait GeneratedTupleConversions {
+  implicit def tuple5[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH] = new PTypeH[(A, B, C, D, E)] {
+    def get(ptf: PTypeFamily) = {
+      ptf.tuple5(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf))
+    }
+  }
+
+  implicit def tuple6[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH] = new PTypeH[(A, B, C, D, E, F)] {
+    def get(ptf: PTypeFamily) = {
+      ptf.tuple6(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf))
+    }
+  }
+
+  implicit def tuple7[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH] = new PTypeH[(A, B, C, D, E, F, G)] {
+    def get(ptf: PTypeFamily) = {
+      ptf.tuple7(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf))
+    }
+  }
+
+  implicit def tuple8[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H)] {
+    def get(ptf: PTypeFamily) = {
+      ptf.tuple8(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf))
+    }
+  }
+
+  implicit def tuple9[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I)] {
+    def get(ptf: PTypeFamily) = {
+      ptf.tuple9(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf))
+    }
+  }
+
+  implicit def tuple10[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J)] {
+    def get(ptf: PTypeFamily) = {
+      ptf.tuple10(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf))
+    }
+  }
+
+  implicit def tuple11[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K)] {
+    def get(ptf: PTypeFamily) = {
+      ptf.tuple11(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf))
+    }
+  }
+
+  implicit def tuple12[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L)] {
+    def get(ptf: PTypeFamily) = {
+      ptf.tuple12(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf))
+    }
+  }
+
+  implicit def tuple13[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M)] {
+    def get(ptf: PTypeFamily) = {
+      ptf.tuple13(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf))
+    }
+  }
+
+  implicit def tuple14[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N)] {
+    def get(ptf: PTypeFamily) = {
+      ptf.tuple14(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf))
+    }
+  }
+
+  implicit def tuple15[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O)] {
+    def get(ptf: PTypeFamily) = {
+      ptf.tuple15(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf))
+    }
+  }
+
+  implicit def tuple16[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH, P: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P)] {
+    def get(ptf: PTypeFamily) = {
+      ptf.tuple16(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf), implicitly[PTypeH[P]].get(ptf))
+    }
+  }
+
+  implicit def tuple17[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH, P: PTypeH, Q: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q)] {
+    def get(ptf: PTypeFamily) = {
+      ptf.tuple17(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf), implicitly[PTypeH[P]].get(ptf), implicitly[PTypeH[Q]].get(ptf))
+    }
+  }
+
+  implicit def tuple18[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH, P: PTypeH, Q: PTypeH, R: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R)] {
+    def get(ptf: PTypeFamily) = {
+      ptf.tuple18(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf), implicitly[PTypeH[P]].get(ptf), implicitly[PTypeH[Q]].get(ptf), implicitly[PTypeH[R]].get(ptf))
+    }
+  }
+
+  implicit def tuple19[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH, P: PTypeH, Q: PTypeH, R: PTypeH, S: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S)] {
+    def get(ptf: PTypeFamily) = {
+      ptf.tuple19(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf), implicitly[PTypeH[P]].get(ptf), implicitly[PTypeH[Q]].get(ptf), implicitly[PTypeH[R]].get(ptf), implicitly[PTypeH[S]].get(ptf))
+    }
+  }
+
+  implicit def tuple20[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH, P: PTypeH, Q: PTypeH, R: PTypeH, S: PTypeH, T: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T)] {
+    def get(ptf: PTypeFamily) = {
+      ptf.tuple20(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf), implicitly[PTypeH[P]].get(ptf), implicitly[PTypeH[Q]].get(ptf), implicitly[PTypeH[R]].get(ptf), implicitly[PTypeH[S]].get(ptf), implicitly[PTypeH[T]].get(ptf))
+    }
+  }
+
+  implicit def tuple21[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH, P: PTypeH, Q: PTypeH, R: PTypeH, S: PTypeH, T: PTypeH, U: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U)] {
+    def get(ptf: PTypeFamily) = {
+      ptf.tuple21(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf), implicitly[PTypeH[P]].get(ptf), implicitly[PTypeH[Q]].get(ptf), implicitly[PTypeH[R]].get(ptf), implicitly[PTypeH[S]].get(ptf), implicitly[PTypeH[T]].get(ptf), implicitly[PTypeH[U]].get(ptf))
+    }
+  }
+
+  implicit def tuple22[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH, P: PTypeH, Q: PTypeH, R: PTypeH, S: PTypeH, T: PTypeH, U: PTypeH, V: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V)] {
+    def get(ptf: PTypeFamily) = {
+      ptf.tuple22(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf), implicitly[PTypeH[P]].get(ptf), implicitly[PTypeH[Q]].get(ptf), implicitly[PTypeH[R]].get(ptf), implicitly[PTypeH[S]].get(ptf), implicitly[PTypeH[T]].get(ptf), implicitly[PTypeH[U]].get(ptf), implicitly[PTypeH[V]].get(ptf))
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/crunch/blob/4693cec0/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTuplePTypeFamily.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTuplePTypeFamily.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTuplePTypeFamily.scala
new file mode 100644
index 0000000..34a424f
--- /dev/null
+++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTuplePTypeFamily.scala
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch.scrunch
+
+import org.apache.crunch.TupleN
+import org.apache.crunch.types.PType
+
+trait GeneratedTuplePTypeFamily extends BasePTypeFamily {
+  import GeneratedTupleHelper._
+
+  def tuple5[A, B, C, D, E](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E]) = {
+    val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E])
+    val out = (t: (A, B, C, D, E)) => tupleN(t._1, t._2, t._3, t._4, t._5)
+    derived(classOf[(A, B, C, D, E)], in, out, ptf.tuples(p0, p1, p2, p3, p4))
+  }
+
+  def tuple6[A, B, C, D, E, F](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F]) = {
+    val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F])
+    val out = (t: (A, B, C, D, E, F)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6)
+    derived(classOf[(A, B, C, D, E, F)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5))
+  }
+
+  def tuple7[A, B, C, D, E, F, G](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G]) = {
+    val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G])
+    val out = (t: (A, B, C, D, E, F, G)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7)
+    derived(classOf[(A, B, C, D, E, F, G)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6))
+  }
+
+  def tuple8[A, B, C, D, E, F, G, H](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H]) = {
+    val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H])
+    val out = (t: (A, B, C, D, E, F, G, H)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8)
+    derived(classOf[(A, B, C, D, E, F, G, H)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7))
+  }
+
+  def tuple9[A, B, C, D, E, F, G, H, I](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I]) = {
+    val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I])
+    val out = (t: (A, B, C, D, E, F, G, H, I)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9)
+    derived(classOf[(A, B, C, D, E, F, G, H, I)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8))
+  }
+
+  def tuple10[A, B, C, D, E, F, G, H, I, J](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J]) = {
+    val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J])
+    val out = (t: (A, B, C, D, E, F, G, H, I, J)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10)
+    derived(classOf[(A, B, C, D, E, F, G, H, I, J)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9))
+  }
+
+  def tuple11[A, B, C, D, E, F, G, H, I, J, K](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K]) = {
+    val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K])
+    val out = (t: (A, B, C, D, E, F, G, H, I, J, K)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11)
+    derived(classOf[(A, B, C, D, E, F, G, H, I, J, K)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10))
+  }
+
+  def tuple12[A, B, C, D, E, F, G, H, I, J, K, L](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L]) = {
+    val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L])
+    val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12)
+    derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11))
+  }
+
+  def tuple13[A, B, C, D, E, F, G, H, I, J, K, L, M](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M]) = {
+    val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M])
+    val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13)
+    derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12))
+  }
+
+  def tuple14[A, B, C, D, E, F, G, H, I, J, K, L, M, N](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N]) = {
+    val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N])
+    val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14)
+    derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13))
+  }
+
+  def tuple15[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O]) = {
+    val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O])
+    val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15)
+    derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14))
+  }
+
+  def tuple16[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O], p15: PType[P]) = {
+    val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O],t.get(15).asInstanceOf[P])
+    val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15, t._16)
+    derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15))
+  }
+
+  def tuple17[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O], p15: PType[P], p16: PType[Q]) = {
+    val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O],t.get(15).asInstanceOf[P],t.get(16).asInstanceOf[Q])
+    val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15, t._16, t._17)
+    derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16))
+  }
+
+  def tuple18[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O], p15: PType[P], p16: PType[Q], p17: PType[R]) = {
+    val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O],t.get(15).asInstanceOf[P],t.get(16).asInstanceOf[Q],t.get(17).asInstanceOf[R])
+    val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15, t._16, t._17, t._18)
+    derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17))
+  }
+
+  def tuple19[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O], p15: PType[P], p16: PType[Q], p17: PType[R], p18: PType[S]) = {
+    val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O],t.get(15).asInstanceOf[P],t.get(16).asInstanceOf[Q],t.get(17).asInstanceOf[R],t.get(18).asInstanceOf[S])
+    val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15, t._16, t._17, t._18, t._19)
+    derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18))
+  }
+
+  def tuple20[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O], p15: PType[P], p16: PType[Q], p17: PType[R], p18: PType[S], p19: PType[T]) = {
+    val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O],t.get(15).asInstanceOf[P],t.get(16).asInstanceOf[Q],t.get(17).asInstanceOf[R],t.get(18).asInstanceOf[S],t.get(19).asInstanceOf[T])
+    val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15, t._16, t._17, t._18, t._19, t._20)
+    derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19))
+  }
+
+  def tuple21[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O], p15: PType[P], p16: PType[Q], p17: PType[R], p18: PType[S], p19: PType[T], p20: PType[U]) = {
+    val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O],t.get(15).asInstanceOf[P],t.get(16).asInstanceOf[Q],t.get(17).asInstanceOf[R],t.get(18).asInstanceOf[S],t.get(19).asInstanceOf[T],t.get(20).asInstanceOf[U])
+    val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15, t._16, t._17, t._18, t._19, t._20, t._21)
+    derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20))
+  }
+
+  def tuple22[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O], p15: PType[P], p16: PType[Q], p17: PType[R], p18: PType[S], p19: PType[T], p20: PType[U], p21: PType[V]) = {
+    val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O],t.get(15).asInstanceOf[P],t.get(16).asInstanceOf[Q],t.get(17).asInstanceOf[R],t.get(18).asInstanceOf[S],t.get(19).asInstanceOf[T],t.get(20).asInstanceOf[U],t.get(21).asInstanceOf[V])
+    val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15, t._16, t._17, t._18, t._19, t._20, t._21, t._22)
+    derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21))
+  }
+
+}
+

http://git-wip-us.apache.org/repos/asf/crunch/blob/4693cec0/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala
index bd03f6f..f77067a 100644
--- a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala
+++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala
@@ -17,7 +17,7 @@
  */
 package org.apache.crunch.scrunch
 
-import org.apache.crunch.{Pair => CPair, Tuple3 => CTuple3, Tuple4 => CTuple4, Union, MapFn}
+import org.apache.crunch.{Pair => CPair, Tuple3 => CTuple3, Tuple4 => CTuple4, TupleN, Union, MapFn}
 import org.apache.crunch.types.{PType, PTypeFamily => PTF}
 import org.apache.crunch.types.writable.{WritableTypeFamily, Writables => CWritables}
 import org.apache.crunch.types.avro.{AvroType, AvroTypeFamily, Avros => CAvros}
@@ -47,10 +47,22 @@ class TMapFn[S, T](val f: S => T, val pt: Option[PType[S]] = None, var init: Boo
   }
 }
 
-trait PTypeFamily {
+object GeneratedTupleHelper {
+  def tupleN(args: Any*) = {
+    TupleN.of(args.map(_.asInstanceOf[AnyRef]): _*)
+  }
+}
 
+trait BasePTypeFamily {
   def ptf: PTF
 
+  def derived[S, T](cls: java.lang.Class[T], in: S => T, out: T => S, pt: PType[S]) = {
+    ptf.derived(cls, new TMapFn[S, T](in, Some(pt)), new TMapFn[T, S](out), pt)
+  }
+}
+
+trait PTypeFamily extends GeneratedTuplePTypeFamily {
+
   def writables[T <: Writable : ClassTag]: PType[T]
 
   def as[T](ptype: PType[T]) = ptf.as(ptype)
@@ -61,10 +73,6 @@ trait PTypeFamily {
 
   def records[T: ClassTag] = ptf.records(implicitly[ClassTag[T]].runtimeClass)
 
-  def derived[S, T](cls: java.lang.Class[T], in: S => T, out: T => S, pt: PType[S]) = {
-    ptf.derived(cls, new TMapFn[S, T](in, Some(pt)), new TMapFn[T, S](out), pt)
-  }
-
   def derivedImmutable[S, T](cls: java.lang.Class[T], in: S => T, out: T => S, pt: PType[S]) = {
     ptf.derivedImmutable(cls, new TMapFn[S, T](in), new TMapFn[T, S](out), pt)
   }

http://git-wip-us.apache.org/repos/asf/crunch/blob/4693cec0/crunch-scrunch/src/test/scala/org/apache/crunch/scrunch/TupleNTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/test/scala/org/apache/crunch/scrunch/TupleNTest.scala b/crunch-scrunch/src/test/scala/org/apache/crunch/scrunch/TupleNTest.scala
new file mode 100644
index 0000000..640a72f
--- /dev/null
+++ b/crunch-scrunch/src/test/scala/org/apache/crunch/scrunch/TupleNTest.scala
@@ -0,0 +1,32 @@
+/*
+ * *
+ *  * Licensed to the Apache Software Foundation (ASF) under one
+ *  * or more contributor license agreements.  See the NOTICE file
+ *  * distributed with this work for additional information
+ *  * regarding copyright ownership.  The ASF licenses this file
+ *  * to you under the Apache License, Version 2.0 (the
+ *  * "License"); you may not use this file except in compliance
+ *  * with the License.  You may obtain a copy of the License at
+ *  *
+ *  *     http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ *
+ */
+
+package org.apache.crunch.scrunch
+
+import org.scalatest.junit.JUnitSuite
+import org.junit.Test
+
+class TupleNTest extends JUnitSuite{
+  @Test def testTupleN {
+    val pc = Mem.collectionOf((1, 2, "a", 3, "b"), (4, 5, "a", 6, "c"))
+    val res = pc.map(x => (x._3, x._4)).groupByKey.combineValues(Aggregators.sum[Int]).materialize
+    org.junit.Assert.assertEquals(List(("a", 9)), res.toList)
+  }
+}


Mime
View raw message