crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject git commit: CRUNCH-403: Use ___ instead of _ as substitute for $ in Avro reflected classes in Scrunch
Date Thu, 29 May 2014 03:13:50 GMT
Repository: crunch
Updated Branches:
  refs/heads/apache-crunch-0.8 8fe96d6cc -> da3ccd9db


CRUNCH-403: Use ___ instead of _ as substitute for $ in Avro reflected classes in Scrunch


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/da3ccd9d
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/da3ccd9d
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/da3ccd9d

Branch: refs/heads/apache-crunch-0.8
Commit: da3ccd9dbcd4bfd8fd834b93283b0051b66dd617
Parents: 8fe96d6
Author: Josh Wills <jwills@apache.org>
Authored: Tue May 27 12:34:11 2014 -0700
Committer: Josh Wills <jwills@apache.org>
Committed: Wed May 28 20:12:06 2014 -0700

----------------------------------------------------------------------
 .../org/apache/crunch/scrunch/PageRankClassTest.scala   | 12 +++++-------
 .../org/apache/crunch/scrunch/ScalaSafeReflectData.java |  4 ++--
 2 files changed, 7 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/da3ccd9d/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala
b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala
index 4cd9e84..3d3cb9f 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala
@@ -27,14 +27,14 @@ import scala.collection.mutable.HashMap
 import _root_.org.junit.Assert._
 import _root_.org.junit.Test
 
-case class PageRankData(pr: Float, oldpr: Float, urls: Array[String]) {
+case class PageRankData(page_rank: Float, oldpr: Float, urls: Array[String]) {
   def this() = this(0f, 0f, null)
 
-  def scaledPageRank = pr / urls.length
+  def scaledPageRank = page_rank / urls.length
 
-  def next(newPageRank: Float) = new PageRankData(newPageRank, pr, urls)
+  def next(newPageRank: Float) = new PageRankData(newPageRank, page_rank, urls)
 
-  def delta = math.abs(pr - oldpr)
+  def delta = math.abs(page_rank - oldpr)
 }
 
 class CachingPageRankClassFn extends DoFn[P[String, PageRankData], P[String, Float]] {
@@ -45,7 +45,7 @@ class CachingPageRankClassFn extends DoFn[P[String, PageRankData], P[String,
Flo
   override def process(input: P[String, PageRankData], emitFn: Emitter[P[String, Float]])
{
     val prd = input.second()
     if (prd.urls.length > 0) {
-      val newpr = prd.pr / prd.urls.length
+      val newpr = prd.page_rank / prd.urls.length
       prd.urls.foreach(url => cache.put(url, cache(url) + newpr))
       if (cache.size > 5000) {
         cleanup(emitFn)
@@ -92,7 +92,6 @@ class PageRankClassTest extends CrunchSuite {
   }
 
   @Test def testPageRank {
-    pipeline.getConfiguration.set("crunch.debug", "true")
     var prev = initialInput(tempDir.copyResourceFileName("urls.txt"))
     var delta = 1.0f
     while (delta > 0.01f) {
@@ -104,7 +103,6 @@ class PageRankClassTest extends CrunchSuite {
   }
 
   def testFastPageRank {
-    pipeline.getConfiguration.set("crunch.debug", "true")
     var prev = initialInput(tempDir.copyResourceFileName("urls.txt"))
     var delta = 1.0f
     while (delta > 0.01f) {

http://git-wip-us.apache.org/repos/asf/crunch/blob/da3ccd9d/crunch-scrunch/src/main/java/org/apache/crunch/scrunch/ScalaSafeReflectData.java
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/main/java/org/apache/crunch/scrunch/ScalaSafeReflectData.java
b/crunch-scrunch/src/main/java/org/apache/crunch/scrunch/ScalaSafeReflectData.java
index 7fbf52f..6118834 100644
--- a/crunch-scrunch/src/main/java/org/apache/crunch/scrunch/ScalaSafeReflectData.java
+++ b/crunch-scrunch/src/main/java/org/apache/crunch/scrunch/ScalaSafeReflectData.java
@@ -236,11 +236,11 @@ public class ScalaSafeReflectData extends ReflectData.AllowNull {
   }
   
   private static String clean(String dirty) {
-    return dirty.replace('$', '_');
+    return dirty.replace("$", "___");
   }
   
   private static String dirty(String clean) {
-    return clean.replace('_', '$');
+    return clean.replace("___", "$");
   }
   
   // Return of this class and its superclasses to serialize.


Mime
View raw message