Return-Path: X-Original-To: apmail-crunch-commits-archive@www.apache.org Delivered-To: apmail-crunch-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 6807010D24 for ; Tue, 18 Feb 2014 01:50:47 +0000 (UTC) Received: (qmail 84303 invoked by uid 500); 18 Feb 2014 01:50:46 -0000 Delivered-To: apmail-crunch-commits-archive@crunch.apache.org Received: (qmail 84263 invoked by uid 500); 18 Feb 2014 01:50:46 -0000 Mailing-List: contact commits-help@crunch.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@crunch.apache.org Delivered-To: mailing list commits@crunch.apache.org Received: (qmail 84256 invoked by uid 99); 18 Feb 2014 01:50:45 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 18 Feb 2014 01:50:45 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 8982482975D; Tue, 18 Feb 2014 01:50:44 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jwills@apache.org To: commits@crunch.apache.org Message-Id: <03c458a73b58471a94c1a806d4fa53f1@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: git commit: CRUNCH-346: Enforce crunch.max.reducers in all cases. Contributed by Jason Gauci. Date: Tue, 18 Feb 2014 01:50:44 +0000 (UTC) Repository: crunch Updated Branches: refs/heads/master bf58906b1 -> 9df580739 CRUNCH-346: Enforce crunch.max.reducers in all cases. Contributed by Jason Gauci. Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/9df58073 Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/9df58073 Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/9df58073 Branch: refs/heads/master Commit: 9df5807397196178ca8328fe5ebb5d98606df062 Parents: bf58906 Author: Josh Wills Authored: Mon Feb 17 17:50:35 2014 -0800 Committer: Josh Wills Committed: Mon Feb 17 17:50:35 2014 -0800 ---------------------------------------------------------------------- .../java/org/apache/crunch/util/PartitionUtils.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/9df58073/crunch-core/src/main/java/org/apache/crunch/util/PartitionUtils.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/util/PartitionUtils.java b/crunch-core/src/main/java/org/apache/crunch/util/PartitionUtils.java index 25f8866..cdcc401 100644 --- a/crunch-core/src/main/java/org/apache/crunch/util/PartitionUtils.java +++ b/crunch-core/src/main/java/org/apache/crunch/util/PartitionUtils.java @@ -37,7 +37,12 @@ public class PartitionUtils { public static int getRecommendedPartitions(PCollection pcollection) { Configuration conf = pcollection.getPipeline().getConfiguration(); - int recommended = getRecommendedPartitions(pcollection, conf); + return getRecommendedPartitions(pcollection, conf); + } + + public static int getRecommendedPartitions(PCollection pcollection, Configuration conf) { + long bytesPerTask = conf.getLong(BYTES_PER_REDUCE_TASK, DEFAULT_BYTES_PER_REDUCE_TASK); + int recommended = 1 + (int) (pcollection.getSize() / bytesPerTask); int maxRecommended = conf.getInt(MAX_REDUCERS, DEFAULT_MAX_REDUCERS); if (maxRecommended > 0 && recommended > maxRecommended) { return maxRecommended; @@ -45,9 +50,4 @@ public class PartitionUtils { return recommended; } } - - public static int getRecommendedPartitions(PCollection pcollection, Configuration conf) { - long bytesPerTask = conf.getLong(BYTES_PER_REDUCE_TASK, DEFAULT_BYTES_PER_REDUCE_TASK); - return 1 + (int) (pcollection.getSize() / bytesPerTask); - } }