Return-Path: X-Original-To: apmail-crunch-commits-archive@www.apache.org Delivered-To: apmail-crunch-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E7527100B7 for ; Thu, 29 May 2014 03:11:50 +0000 (UTC) Received: (qmail 7048 invoked by uid 500); 29 May 2014 03:11:50 -0000 Delivered-To: apmail-crunch-commits-archive@crunch.apache.org Received: (qmail 7009 invoked by uid 500); 29 May 2014 03:11:50 -0000 Mailing-List: contact commits-help@crunch.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@crunch.apache.org Delivered-To: mailing list commits@crunch.apache.org Received: (qmail 7002 invoked by uid 99); 29 May 2014 03:11:50 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 29 May 2014 03:11:50 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id E17634EE40; Thu, 29 May 2014 03:11:49 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jwills@apache.org To: commits@crunch.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: git commit: CRUNCH-403: Use ___ instead of _ as substitute for $ in Avro reflected classes in Scrunch Date: Thu, 29 May 2014 03:11:49 +0000 (UTC) Repository: crunch Updated Branches: refs/heads/master 077343a5d -> a62a24b6f CRUNCH-403: Use ___ instead of _ as substitute for $ in Avro reflected classes in Scrunch Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/a62a24b6 Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/a62a24b6 Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/a62a24b6 Branch: refs/heads/master Commit: a62a24b6f80666d92cac2b6dfa5c088e51f26ef6 Parents: 077343a Author: Josh Wills Authored: Tue May 27 12:34:11 2014 -0700 Committer: Josh Wills Committed: Wed May 28 20:07:36 2014 -0700 ---------------------------------------------------------------------- .../org/apache/crunch/scrunch/PageRankClassTest.scala | 12 +++++------- .../org/apache/crunch/scrunch/ScalaSafeReflectData.java | 4 ++-- 2 files changed, 7 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/a62a24b6/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala index 4cd9e84..3d3cb9f 100644 --- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala +++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala @@ -27,14 +27,14 @@ import scala.collection.mutable.HashMap import _root_.org.junit.Assert._ import _root_.org.junit.Test -case class PageRankData(pr: Float, oldpr: Float, urls: Array[String]) { +case class PageRankData(page_rank: Float, oldpr: Float, urls: Array[String]) { def this() = this(0f, 0f, null) - def scaledPageRank = pr / urls.length + def scaledPageRank = page_rank / urls.length - def next(newPageRank: Float) = new PageRankData(newPageRank, pr, urls) + def next(newPageRank: Float) = new PageRankData(newPageRank, page_rank, urls) - def delta = math.abs(pr - oldpr) + def delta = math.abs(page_rank - oldpr) } class CachingPageRankClassFn extends DoFn[P[String, PageRankData], P[String, Float]] { @@ -45,7 +45,7 @@ class CachingPageRankClassFn extends DoFn[P[String, PageRankData], P[String, Flo override def process(input: P[String, PageRankData], emitFn: Emitter[P[String, Float]]) { val prd = input.second() if (prd.urls.length > 0) { - val newpr = prd.pr / prd.urls.length + val newpr = prd.page_rank / prd.urls.length prd.urls.foreach(url => cache.put(url, cache(url) + newpr)) if (cache.size > 5000) { cleanup(emitFn) @@ -92,7 +92,6 @@ class PageRankClassTest extends CrunchSuite { } @Test def testPageRank { - pipeline.getConfiguration.set("crunch.debug", "true") var prev = initialInput(tempDir.copyResourceFileName("urls.txt")) var delta = 1.0f while (delta > 0.01f) { @@ -104,7 +103,6 @@ class PageRankClassTest extends CrunchSuite { } def testFastPageRank { - pipeline.getConfiguration.set("crunch.debug", "true") var prev = initialInput(tempDir.copyResourceFileName("urls.txt")) var delta = 1.0f while (delta > 0.01f) { http://git-wip-us.apache.org/repos/asf/crunch/blob/a62a24b6/crunch-scrunch/src/main/java/org/apache/crunch/scrunch/ScalaSafeReflectData.java ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/main/java/org/apache/crunch/scrunch/ScalaSafeReflectData.java b/crunch-scrunch/src/main/java/org/apache/crunch/scrunch/ScalaSafeReflectData.java index 7fbf52f..6118834 100644 --- a/crunch-scrunch/src/main/java/org/apache/crunch/scrunch/ScalaSafeReflectData.java +++ b/crunch-scrunch/src/main/java/org/apache/crunch/scrunch/ScalaSafeReflectData.java @@ -236,11 +236,11 @@ public class ScalaSafeReflectData extends ReflectData.AllowNull { } private static String clean(String dirty) { - return dirty.replace('$', '_'); + return dirty.replace("$", "___"); } private static String dirty(String clean) { - return clean.replace('_', '$'); + return clean.replace("___", "$"); } // Return of this class and its superclasses to serialize.