crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject crunch git commit: CRUNCH-508: Cache Enumeration.toString values in Scrunch
Date Thu, 23 Apr 2015 13:49:29 GMT
Repository: crunch
Updated Branches:
  refs/heads/master b34cf6dfb -> ba7a3477c


CRUNCH-508: Cache Enumeration.toString values in Scrunch


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/ba7a3477
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/ba7a3477
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/ba7a3477

Branch: refs/heads/master
Commit: ba7a3477cd74777717c19cc0a85916a2a2208e8b
Parents: b34cf6d
Author: Josh Wills <jwills@apache.org>
Authored: Tue Apr 7 11:48:02 2015 -0700
Committer: Josh Wills <jwills@apache.org>
Committed: Thu Apr 23 08:49:09 2015 -0500

----------------------------------------------------------------------
 .../apache/crunch/scrunch/IncrementTest.scala    |  9 +++++++++
 .../org/apache/crunch/scrunch/Increment.scala    | 19 +++++++++++++++++--
 2 files changed, 26 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/ba7a3477/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/IncrementTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/IncrementTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/IncrementTest.scala
index 338051c..44aa9a8 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/IncrementTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/IncrementTest.scala
@@ -22,8 +22,15 @@ import org.apache.crunch.io.{From => from, To => to}
 import _root_.org.junit.Test
 import _root_.org.junit.Assert.assertEquals
 
+
 class IncrementTest extends CrunchSuite {
 
+  object Inc extends Enumeration {
+    type Inc = Value
+    val A, B, C, D = Value
+  }
+  import Inc._
+
   @Test def testIncrement {
     val pipeline = Pipeline.mapReduce[IncrementTest](tempDir.getDefaultConfiguration)
     val input = tempDir.copyResourceFileName("shakes.txt")
@@ -34,6 +41,7 @@ class IncrementTest extends CrunchSuite {
         .filter(!_.isEmpty())
         .increment("TOP", "NONEMPTY")
         .incrementIf(_ startsWith "a")("TOP", "AWORDS_2x", 2)
+        .increment(Inc, Inc.A)
         .write(to.avroFile(tempDir.getFileName("somewords")))
 
     val res = pipeline.done()
@@ -41,5 +49,6 @@ class IncrementTest extends CrunchSuite {
     assertEquals(21836, sr0.getCounterValue("TOP", "ALLWORDS"))
     assertEquals(20366, sr0.getCounterValue("TOP", "NONEMPTY"))
     assertEquals(3604, sr0.getCounterValue("TOP", "AWORDS_2x"))
+    assertEquals(20366, sr0.getCounterValue("Inc", "A"))
   }
 }

http://git-wip-us.apache.org/repos/asf/crunch/blob/ba7a3477/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Increment.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Increment.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Increment.scala
index f6b6ffa..046c777 100644
--- a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Increment.scala
+++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Increment.scala
@@ -29,6 +29,9 @@ import org.apache.crunch.{FilterFn, Pair => CPair}
  * {@code Incrementable} trait.
  */
 trait Incrementable[T] {
+
+  private val enumCache = new java.util.HashMap[Enumeration, String]()
+
   def increment(counter: Enum[_]): T = {
     increment(counter, 1)
   }
@@ -42,7 +45,12 @@ trait Incrementable[T] {
   }
 
   def increment(groupEnum: Enumeration, value: Enumeration#Value, count: Long): T = {
-    increment(groupEnum.toString, value.toString, count)
+    var groupName = enumCache.get(groupEnum)
+    if (groupName == null) {
+      groupName = groupEnum.toString
+      enumCache.put(groupEnum, groupName)
+    }
+    increment(groupName, value.toString, count)
   }
 
   def increment(groupName: String, counterName: String): T = {
@@ -61,6 +69,8 @@ trait Incrementable[T] {
  */
 trait Increment[T] {
 
+  private val enumCache = new java.util.HashMap[Enumeration, String]()
+
   def apply(counter: Enum[_]): T = {
     apply(counter, 1)
   }
@@ -74,7 +84,12 @@ trait Increment[T] {
   }
 
   def apply(groupEnum: Enumeration, value: Enumeration#Value, count: Long): T = {
-    apply(groupEnum.toString, value.toString, count)
+    var groupName = enumCache.get(groupEnum)
+    if (groupName == null) {
+      groupName = groupEnum.toString
+      enumCache.put(groupEnum, groupName)
+    }
+    apply(groupName, value.toString, count)
   }
 
   def apply(groupName: String, counterName: String): T = {


Mime
View raw message