Return-Path: 
 <chukwa-commits-return-777-apmail-incubator-chukwa-commits-archive=incubator.apache.org@incubator.apache.org>
X-Original-To: apmail-incubator-chukwa-commits-archive@www.apache.org
Delivered-To: apmail-incubator-chukwa-commits-archive@www.apache.org
Received: from mail.apache.org (hermes.apache.org [140.211.11.3])
	by minotaur.apache.org (Postfix) with SMTP id B951DD6E2
	for <apmail-incubator-chukwa-commits-archive@www.apache.org>;
 Tue, 17 Jul 2012 00:06:58 +0000 (UTC)
Received: (qmail 10211 invoked by uid 500); 17 Jul 2012 00:06:58 -0000
Delivered-To: apmail-incubator-chukwa-commits-archive@incubator.apache.org
Received: (qmail 10185 invoked by uid 500); 17 Jul 2012 00:06:58 -0000
Mailing-List: contact chukwa-commits-help@incubator.apache.org; run by ezmlm
Precedence: bulk
List-Help: <mailto:chukwa-commits-help@incubator.apache.org>
List-Unsubscribe: <mailto:chukwa-commits-unsubscribe@incubator.apache.org>
List-Post: <mailto:chukwa-commits@incubator.apache.org>
List-Id: <chukwa-commits.incubator.apache.org>
Reply-To: chukwa-dev@incubator.apache.org
Delivered-To: mailing list chukwa-commits@incubator.apache.org
Received: (qmail 10175 invoked by uid 99); 17 Jul 2012 00:06:58 -0000
Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136)
    by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 17 Jul 2012 00:06:58 +0000
X-ASF-Spam-Status: No, hits=-2000.0 required=5.0
	tests=ALL_TRUSTED
X-Spam-Check-By: apache.org
Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4)
    by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 17 Jul 2012 00:06:56 +0000
Received: from eris.apache.org (localhost [127.0.0.1])
	by eris.apache.org (Postfix) with ESMTP id 9DAB7238889B;
	Tue, 17 Jul 2012 00:06:37 +0000 (UTC)
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1362318 - in
 /incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction:
 CHUKWA_CONSTANT.java demux/ChukwaRecordPartitioner.java
Date: Tue, 17 Jul 2012 00:06:37 -0000
To: chukwa-commits@incubator.apache.org
From: asrabkin@apache.org
X-Mailer: svnmailer-1.0.8-patched
Message-Id: <20120717000637.9DAB7238889B@eris.apache.org>
X-Virus-Checked: Checked by ClamAV on apache.org

Author: asrabkin
Date: Tue Jul 17 00:06:37 2012
New Revision: 1362318

URL: http://svn.apache.org/viewvc?rev=1362318&view=rev
Log:
CHUKWA-647. Spread out intermediate data with the same ReduceType into different Reduce Tasks. Contributed by Jie Huang.

Modified:
    incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/CHUKWA_CONSTANT.java
    incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/ChukwaRecordPartitioner.java

Modified: incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/CHUKWA_CONSTANT.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/CHUKWA_CONSTANT.java?rev=1362318&r1=1362317&r2=1362318&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/CHUKWA_CONSTANT.java (original)
+++ incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/CHUKWA_CONSTANT.java Tue Jul 17 00:06:37 2012
@@ -64,5 +64,6 @@ public interface CHUKWA_CONSTANT {
   public static final String ARCHIVES_IN_ERROR_DIR_NAME      = "inError/";
 
   public static final String POST_DEMUX_DATA_LOADER = "chukwa.post.demux.data.loader";  
+  public static final String INCLUDE_KEY_IN_PARTITIONER = "_";
   public static final String POST_DEMUX_SUCCESS_ACTION = "chukwa.post.demux.success.action";  
 }

Modified: incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/ChukwaRecordPartitioner.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/ChukwaRecordPartitioner.java?rev=1362318&r1=1362317&r2=1362318&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/ChukwaRecordPartitioner.java (original)
+++ incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/ChukwaRecordPartitioner.java Tue Jul 17 00:06:37 2012
@@ -21,6 +21,8 @@ package org.apache.hadoop.chukwa.extract
 
 import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecord;
 import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey;
+import org.apache.hadoop.chukwa.extraction.CHUKWA_CONSTANT;
+
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.Partitioner;
 import org.apache.log4j.Logger;
@@ -44,7 +46,9 @@ public class ChukwaRecordPartitioner<K, 
               + "] - Reducer:"
               + ((key.getReduceType().hashCode() & Integer.MAX_VALUE) % numReduceTasks));
     }
-    return (key.getReduceType().hashCode() & Integer.MAX_VALUE)
+    String hashkey = key.getReduceType();
+    if(key.getKey().startsWith(CHUKWA_CONSTANT.INCLUDE_KEY_IN_PARTITIONER)) hashkey = key.getReduceType()+"#"+key.getKey();
+    return (hashkey.hashCode() & Integer.MAX_VALUE)
         % numReduceTasks;
   }