hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sha...@apache.org
Subject svn commit: r788628 - in /hadoop/mapreduce/trunk: CHANGES.txt src/java/org/apache/hadoop/mapred/lib/KeyFieldBasedPartitioner.java src/test/mapred/org/apache/hadoop/mapred/lib/TestKeyFieldBasedPartitioner.java
Date Fri, 26 Jun 2009 08:37:00 GMT
Author: sharad
Date: Fri Jun 26 08:37:00 2009
New Revision: 788628

URL: http://svn.apache.org/viewvc?rev=788628&view=rev
Log:
MAPREDUCE-2. Fixes a bug in KeyFieldBasedPartitioner in handling empty keys. Contributed by
Amar Kamat.

Added:
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/lib/TestKeyFieldBasedPartitioner.java
Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/lib/KeyFieldBasedPartitioner.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=788628&r1=788627&r2=788628&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Fri Jun 26 08:37:00 2009
@@ -39,3 +39,6 @@
 
     MAPREDUCE-419. Reconcile mapred.userlog.limit.kb defaults in configuration
     and code. (Philip Zeyliger via cdouglas)
+
+    MAPREDUCE-2. Fixes a bug in KeyFieldBasedPartitioner in handling empty
+    keys. (Amar Kamat via sharad)

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/lib/KeyFieldBasedPartitioner.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/lib/KeyFieldBasedPartitioner.java?rev=788628&r1=788627&r2=788628&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/lib/KeyFieldBasedPartitioner.java
(original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/lib/KeyFieldBasedPartitioner.java
Fri Jun 26 08:37:00 2009
@@ -76,12 +76,21 @@
       throw new RuntimeException("The current system does not " +
           "support UTF-8 encoding!", e);
     }
+    // return 0 if the key is empty
+    if (keyBytes.length == 0) {
+      return 0;
+    }
+    
     int []lengthIndicesFirst = keyFieldHelper.getWordLengths(keyBytes, 0, 
         keyBytes.length);
     int currentHash = 0;
     for (KeyDescription keySpec : allKeySpecs) {
       int startChar = keyFieldHelper.getStartOffset(keyBytes, 0, keyBytes.length, 
           lengthIndicesFirst, keySpec);
+       // no key found! continue
+      if (startChar < 0) {
+        continue;
+      }
       int endChar = keyFieldHelper.getEndOffset(keyBytes, 0, keyBytes.length, 
           lengthIndicesFirst, keySpec);
       currentHash = hashCode(keyBytes, startChar, endChar, 

Added: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/lib/TestKeyFieldBasedPartitioner.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/lib/TestKeyFieldBasedPartitioner.java?rev=788628&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/lib/TestKeyFieldBasedPartitioner.java
(added)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/lib/TestKeyFieldBasedPartitioner.java
Fri Jun 26 08:37:00 2009
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.lib;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner;
+
+import junit.framework.TestCase;
+
+public class TestKeyFieldBasedPartitioner extends TestCase {
+
+  /**
+   * Test is key-field-based partitioned works with empty key.
+   */
+  public void testEmptyKey() throws Exception {
+    KeyFieldBasedPartitioner<Text, Text> kfbp = 
+      new KeyFieldBasedPartitioner<Text, Text>();
+    JobConf conf = new JobConf();
+    conf.setInt("num.key.fields.for.partition", 10);
+    kfbp.configure(conf);
+    assertEquals("Empty key should map to 0th partition", 
+                 0, kfbp.getPartition(new Text(), new Text(), 10));
+  }
+}
\ No newline at end of file



Mime
View raw message