hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1076966 - in /hadoop/common/branches/branch-0.20-security-patches/src: mapred/org/apache/hadoop/mapred/lib/ test/org/apache/hadoop/mapred/lib/
Date Fri, 04 Mar 2011 03:26:35 GMT
Author: omalley
Date: Fri Mar  4 03:26:34 2011
New Revision: 1076966

URL: http://svn.apache.org/viewvc?rev=1076966&view=rev
Log:
commit 4640e24bd88304f22968e1433ad5804a12192506
Author: Yahoo\! <ltucker@yahoo-inc.com>
Date:   Tue Aug 11 10:07:23 2009 -0700

    Walk back the 0.20 branch to the correct point
    Revert "MAPREDUCE-735. Fixes a problem in the KeyFieldHelper to do with the end index
for some inputs. Contributed by Amar Kamat."
    
    This reverts commit 66a3d58e6f3ca322bac445739021e765009e677b.

Removed:
    hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/lib/TestKeyFieldHelper.java
Modified:
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/lib/KeyFieldBasedPartitioner.java
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/lib/KeyFieldHelper.java
    hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java
    hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/lib/TestKeyFieldBasedPartitioner.java

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/lib/KeyFieldBasedPartitioner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/lib/KeyFieldBasedPartitioner.java?rev=1076966&r1=1076965&r2=1076966&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/lib/KeyFieldBasedPartitioner.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/lib/KeyFieldBasedPartitioner.java
Fri Mar  4 03:26:34 2011
@@ -67,7 +67,7 @@ public class KeyFieldBasedPartitioner<K2
 
     List <KeyDescription> allKeySpecs = keyFieldHelper.keySpecs();
     if (allKeySpecs.size() == 0) {
-      return getPartition(key.toString().hashCode(), numReduceTasks);
+      return (key.toString().hashCode() & Integer.MAX_VALUE) % numReduceTasks;
     }
 
     try {
@@ -96,7 +96,7 @@ public class KeyFieldBasedPartitioner<K2
       currentHash = hashCode(keyBytes, startChar, endChar, 
           currentHash);
     }
-    return getPartition(currentHash, numReduceTasks);
+    return (currentHash & Integer.MAX_VALUE) % numReduceTasks;
   }
   
   protected int hashCode(byte[] b, int start, int end, int currentHash) {
@@ -106,7 +106,4 @@ public class KeyFieldBasedPartitioner<K2
     return currentHash;
   }
 
-  protected int getPartition(int hash, int numReduceTasks) {
-    return (hash & Integer.MAX_VALUE) % numReduceTasks;
-  }
 }

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/lib/KeyFieldHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/lib/KeyFieldHelper.java?rev=1076966&r1=1076965&r2=1076966&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/lib/KeyFieldHelper.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/lib/KeyFieldHelper.java
Fri Mar  4 03:26:34 2011
@@ -47,13 +47,6 @@ class KeyFieldHelper {
     int endChar = 0;
     boolean numeric;
     boolean reverse;
-    @Override
-    public String toString() {
-      return "-k" 
-             + beginFieldIdx + "." + beginChar + "," 
-             + endFieldIdx + "." + endChar 
-             + (numeric ? "n" : "") + (reverse ? "r" : "");
-    }
   }
   
   private List<KeyDescription> allKeySpecs = new ArrayList<KeyDescription>();
@@ -137,7 +130,7 @@ class KeyFieldHelper {
     if (k.endFieldIdx == 0) {
       //there is no end field specified for this keyspec. So the remaining
       //part of the key is considered in its entirety.
-      return end - 1; 
+      return end; 
     }
     if (lengthIndices[0] >= k.endFieldIdx) {
       int position = 0;
@@ -151,9 +144,9 @@ class KeyFieldHelper {
       if (position + k.endChar <= (end - start)) {
         return start + position + k.endChar - 1;
       }
-      return end - 1;
+      return end;
     }
-    return end - 1;
+    return end;
   }
   public void parseOption(String option) {
     if (option == null || option.equals("")) {

Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java?rev=1076966&r1=1076965&r2=1076966&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java
Fri Mar  4 03:26:34 2011
@@ -133,8 +133,6 @@ public class TestKeyFieldBasedComparator
     configure("-k10,10",2);
     
     localTestWithoutMRJob("-k9,9", 1);
-
-    localTestWithoutMRJob("-k9n", 1);
   }
   
   byte[] line1_bytes = line1.getBytes();

Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/lib/TestKeyFieldBasedPartitioner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/lib/TestKeyFieldBasedPartitioner.java?rev=1076966&r1=1076965&r2=1076966&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/lib/TestKeyFieldBasedPartitioner.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/lib/TestKeyFieldBasedPartitioner.java
Fri Mar  4 03:26:34 2011
@@ -29,98 +29,12 @@ public class TestKeyFieldBasedPartitione
    * Test is key-field-based partitioned works with empty key.
    */
   public void testEmptyKey() throws Exception {
-    int numReducers = 10;
     KeyFieldBasedPartitioner<Text, Text> kfbp = 
       new KeyFieldBasedPartitioner<Text, Text>();
     JobConf conf = new JobConf();
     conf.setInt("num.key.fields.for.partition", 10);
     kfbp.configure(conf);
     assertEquals("Empty key should map to 0th partition", 
-                 0, kfbp.getPartition(new Text(), new Text(), numReducers));
-    
-    // check if the hashcode is correct when no keyspec is specified
-    kfbp = new KeyFieldBasedPartitioner<Text, Text>();
-    conf = new JobConf();
-    kfbp.configure(conf);
-    String input = "abc\tdef\txyz";
-    int hashCode = input.hashCode();
-    int expectedPartition = kfbp.getPartition(hashCode, numReducers);
-    assertEquals("Partitioner doesnt work as expected", expectedPartition, 
-                 kfbp.getPartition(new Text(input), new Text(), numReducers));
-    
-    // check if the hashcode is correct with specified keyspec
-    kfbp = new KeyFieldBasedPartitioner<Text, Text>();
-    conf = new JobConf();
-    conf.set("mapred.text.key.partitioner.options", "-k2,2");
-    kfbp.configure(conf);
-    String expectedOutput = "def";
-    byte[] eBytes = expectedOutput.getBytes();
-    hashCode = kfbp.hashCode(eBytes, 0, eBytes.length - 1, 0);
-    expectedPartition = kfbp.getPartition(hashCode, numReducers);
-    assertEquals("Partitioner doesnt work as expected", expectedPartition, 
-                 kfbp.getPartition(new Text(input), new Text(), numReducers));
-    
-    // test with invalid end index in keyspecs
-    kfbp = new KeyFieldBasedPartitioner<Text, Text>();
-    conf = new JobConf();
-    conf.set("mapred.text.key.partitioner.options", "-k2,5");
-    kfbp.configure(conf);
-    expectedOutput = "def\txyz";
-    eBytes = expectedOutput.getBytes();
-    hashCode = kfbp.hashCode(eBytes, 0, eBytes.length - 1, 0);
-    expectedPartition = kfbp.getPartition(hashCode, numReducers);
-    assertEquals("Partitioner doesnt work as expected", expectedPartition, 
-                 kfbp.getPartition(new Text(input), new Text(), numReducers));
-    
-    // test with 0 end index in keyspecs
-    kfbp = new KeyFieldBasedPartitioner<Text, Text>();
-    conf = new JobConf();
-    conf.set("mapred.text.key.partitioner.options", "-k2");
-    kfbp.configure(conf);
-    expectedOutput = "def\txyz";
-    eBytes = expectedOutput.getBytes();
-    hashCode = kfbp.hashCode(eBytes, 0, eBytes.length - 1, 0);
-    expectedPartition = kfbp.getPartition(hashCode, numReducers);
-    assertEquals("Partitioner doesnt work as expected", expectedPartition, 
-                 kfbp.getPartition(new Text(input), new Text(), numReducers));
-    
-    // test with invalid keyspecs
-    kfbp = new KeyFieldBasedPartitioner<Text, Text>();
-    conf = new JobConf();
-    conf.set("mapred.text.key.partitioner.options", "-k10");
-    kfbp.configure(conf);
-    assertEquals("Partitioner doesnt work as expected", 0, 
-                 kfbp.getPartition(new Text(input), new Text(), numReducers));
-    
-    // test with multiple keyspecs
-    kfbp = new KeyFieldBasedPartitioner<Text, Text>();
-    conf = new JobConf();
-    conf.set("mapred.text.key.partitioner.options", "-k2,2 -k4,4");
-    kfbp.configure(conf);
-    input = "abc\tdef\tpqr\txyz";
-    expectedOutput = "def";
-    eBytes = expectedOutput.getBytes();
-    hashCode = kfbp.hashCode(eBytes, 0, eBytes.length - 1, 0);
-    expectedOutput = "xyz";
-    eBytes = expectedOutput.getBytes();
-    hashCode = kfbp.hashCode(eBytes, 0, eBytes.length - 1, hashCode);
-    expectedPartition = kfbp.getPartition(hashCode, numReducers);
-    assertEquals("Partitioner doesnt work as expected", expectedPartition, 
-                 kfbp.getPartition(new Text(input), new Text(), numReducers));
-    
-    // test with invalid start index in keyspecs
-    kfbp = new KeyFieldBasedPartitioner<Text, Text>();
-    conf = new JobConf();
-    conf.set("mapred.text.key.partitioner.options", "-k2,2 -k30,21 -k4,4 -k5");
-    kfbp.configure(conf);
-    expectedOutput = "def";
-    eBytes = expectedOutput.getBytes();
-    hashCode = kfbp.hashCode(eBytes, 0, eBytes.length - 1, 0);
-    expectedOutput = "xyz";
-    eBytes = expectedOutput.getBytes();
-    hashCode = kfbp.hashCode(eBytes, 0, eBytes.length - 1, hashCode);
-    expectedPartition = kfbp.getPartition(hashCode, numReducers);
-    assertEquals("Partitioner doesnt work as expected", expectedPartition, 
-                 kfbp.getPartition(new Text(input), new Text(), numReducers));
+                 0, kfbp.getPartition(new Text(), new Text(), 10));
   }
-}
+}
\ No newline at end of file



Mime
View raw message