crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject crunch git commit: CRUNCH-525: Correct (more) accurate default scale factors for built-in MapFn implementations
Date Fri, 22 May 2015 18:40:41 GMT
Repository: crunch
Updated Branches:
  refs/heads/master 85b985a72 -> 9bb526fd1


CRUNCH-525: Correct (more) accurate default scale factors for built-in MapFn implementations


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/9bb526fd
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/9bb526fd
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/9bb526fd

Branch: refs/heads/master
Commit: 9bb526fd1f67c06641c24e29a3e2b1078877e9f2
Parents: 85b985a
Author: Josh Wills <jwills@apache.org>
Authored: Thu May 21 17:39:02 2015 -0700
Committer: Josh Wills <jwills@apache.org>
Committed: Fri May 22 11:21:21 2015 -0700

----------------------------------------------------------------------
 .../src/main/java/org/apache/crunch/fn/CompositeMapFn.java    | 5 +++++
 .../src/main/java/org/apache/crunch/fn/ExtractKeyFn.java      | 5 +++++
 crunch-core/src/main/java/org/apache/crunch/fn/PairMapFn.java | 7 ++++++-
 3 files changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/9bb526fd/crunch-core/src/main/java/org/apache/crunch/fn/CompositeMapFn.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/fn/CompositeMapFn.java b/crunch-core/src/main/java/org/apache/crunch/fn/CompositeMapFn.java
index 8c63370..69dfa7e 100644
--- a/crunch-core/src/main/java/org/apache/crunch/fn/CompositeMapFn.java
+++ b/crunch-core/src/main/java/org/apache/crunch/fn/CompositeMapFn.java
@@ -74,4 +74,9 @@ public class CompositeMapFn<R, S, T> extends MapFn<R, T> {
     first.configure(conf);
     second.configure(conf);
   }
+
+  @Override
+  public float scaleFactor() {
+    return first.scaleFactor() * second.scaleFactor();
+  }
 }

http://git-wip-us.apache.org/repos/asf/crunch/blob/9bb526fd/crunch-core/src/main/java/org/apache/crunch/fn/ExtractKeyFn.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/fn/ExtractKeyFn.java b/crunch-core/src/main/java/org/apache/crunch/fn/ExtractKeyFn.java
index 7089ebf..2d2776e 100644
--- a/crunch-core/src/main/java/org/apache/crunch/fn/ExtractKeyFn.java
+++ b/crunch-core/src/main/java/org/apache/crunch/fn/ExtractKeyFn.java
@@ -55,6 +55,11 @@ public class ExtractKeyFn<K, V> extends MapFn<V, Pair<K, V>>
{
   }
 
   @Override
+  public float scaleFactor() {
+    return 1.0f + mapFn.scaleFactor();
+  }
+
+  @Override
   public Pair<K, V> map(V input) {
     return Pair.of(mapFn.map(input), input);
   }

http://git-wip-us.apache.org/repos/asf/crunch/blob/9bb526fd/crunch-core/src/main/java/org/apache/crunch/fn/PairMapFn.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/fn/PairMapFn.java b/crunch-core/src/main/java/org/apache/crunch/fn/PairMapFn.java
index cdb1ecf..c8d55e0 100644
--- a/crunch-core/src/main/java/org/apache/crunch/fn/PairMapFn.java
+++ b/crunch-core/src/main/java/org/apache/crunch/fn/PairMapFn.java
@@ -56,7 +56,12 @@ public class PairMapFn<K, V, S, T> extends MapFn<Pair<K, V>,
Pair<S, T>> {
     keys.initialize();
     values.initialize();
   }
-  
+
+  @Override
+  public float scaleFactor() {
+    return (keys.scaleFactor() + values.scaleFactor()) / 2.0f;
+  }
+
   @Override
   public Pair<S, T> map(Pair<K, V> input) {
     return Pair.of(keys.map(input.first()), values.map(input.second()));


Mime
View raw message