crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject crunch git commit: CRUNCH-511: Switch case class PTypes to use derived instead of derivedImmutable
Date Sat, 18 Apr 2015 15:22:01 GMT
Repository: crunch
Updated Branches:
  refs/heads/master 3ab0b078c -> b34cf6dfb


CRUNCH-511: Switch case class PTypes to use derived instead of derivedImmutable


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/b34cf6df
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/b34cf6df
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/b34cf6df

Branch: refs/heads/master
Commit: b34cf6dfbd4875fed7762fea10e94ddda52c5ae3
Parents: 3ab0b07
Author: Josh Wills <jwills@apache.org>
Authored: Thu Apr 16 17:34:33 2015 -0400
Committer: Josh Wills <jwills@apache.org>
Committed: Sat Apr 18 08:12:55 2015 -0700

----------------------------------------------------------------------
 .../crunch/scrunch/JoinCaseClassKeyTest.scala   | 46 ++++++++++++++++++++
 .../org/apache/crunch/scrunch/PTypeFamily.scala |  2 +-
 2 files changed, 47 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/b34cf6df/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/JoinCaseClassKeyTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/JoinCaseClassKeyTest.scala
b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/JoinCaseClassKeyTest.scala
new file mode 100644
index 0000000..c97b2c6
--- /dev/null
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/JoinCaseClassKeyTest.scala
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch.scrunch
+
+import java.nio.ByteBuffer
+
+import _root_.org.junit.Test
+
+case class AKey(a: String, b: ByteBuffer)
+case class Keyed1(key: AKey, value: String)
+case class Keyed2(key: AKey, value: Int)
+
+class JoinCaseClassKeyTest extends CrunchSuite {
+
+  lazy val pipeline = Pipeline.mapReduce[JoinCaseClassKeyTest](tempDir.getDefaultConfiguration)
+
+  @Test def testCaseClassJoin: Unit = {
+    val bb1 = ByteBuffer.wrap(Array[Byte](1, 2, 0))
+    val bb2 = ByteBuffer.wrap(Array[Byte](1, 2, 0))
+    val bb3 = ByteBuffer.wrap(Array[Byte](1, 0, 3))
+    val bb4 = ByteBuffer.wrap(Array[Byte](1, 0, 3))
+    val k = Array(AKey("a", bb1), AKey("a", bb2), AKey("a", bb3), AKey("a", bb4))
+    val ones = Seq(Keyed1(k(0), "x"), Keyed1(k(2), "r"))
+    val twos = Seq(Keyed2(k(1), 1), Keyed2(k(3), 2))
+
+    val o = pipeline.create(ones, Avros.caseClasses[Keyed1])
+    val t = pipeline.create(twos, Avros.caseClasses[Keyed2])
+    val joined = o.by(x => x.key).join(t.by(x => x.key))
+    assert(2 == joined.materialize().seq.size)
+  }
+}

http://git-wip-us.apache.org/repos/asf/crunch/blob/b34cf6df/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala
index 19c9421..95ff25b 100644
--- a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala
+++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala
@@ -242,7 +242,7 @@ trait PTypeFamily extends GeneratedTuplePTypeFamily {
     val out = (x: Product) => TupleN.of(x.productIterator.toArray.asInstanceOf[Array[Object]]
: _*)
     val rtc = currentMirror.runtimeClass(tpe)
     val base = namedTuples(rtc.getCanonicalName + "_", args) // See CRUNCH-495
-    ptf.derivedImmutable(rtc.asInstanceOf[Class[T]], new TypeMapFn[T](rtc), new TMapFn[T,
TupleN](out), base)
+    ptf.derived(rtc.asInstanceOf[Class[T]], new TypeMapFn[T](rtc), new TMapFn[T, TupleN](out),
base)
   }
 
   private val classToPrimitivePType = Map(


Mime
View raw message