pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1211641 - in /pig/branches/branch-0.10: ./ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/ src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/ src/org/apache/pig/backend/hadoo...
Date Wed, 07 Dec 2011 21:21:09 GMT
Author: daijy
Date: Wed Dec  7 21:21:08 2011
New Revision: 1211641

URL: http://svn.apache.org/viewvc?rev=1211641&view=rev
Log:
PIG-2370: SkewedParitioner results in Kerberos error

Modified:
    pig/branches/branch-0.10/CHANGES.txt
    pig/branches/branch-0.10/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/SkewedPartitioner.java
    pig/branches/branch-0.10/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartitionRearrange.java
    pig/branches/branch-0.10/src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java

Modified: pig/branches/branch-0.10/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/CHANGES.txt?rev=1211641&r1=1211640&r2=1211641&view=diff
==============================================================================
--- pig/branches/branch-0.10/CHANGES.txt (original)
+++ pig/branches/branch-0.10/CHANGES.txt Wed Dec  7 21:21:08 2011
@@ -296,6 +296,8 @@ Release 0.9.2 - Unreleased
 
 BUG FIXES
 
+PIG-2370: SkewedParitioner results in Kerberos error (daijy)
+
 PIG-2374: streaming regression with dotNext (daijy)
 
 PIG-2387: BinStorageRecordReader causes negative progress (xutingz via daijy)

Modified: pig/branches/branch-0.10/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/SkewedPartitioner.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/SkewedPartitioner.java?rev=1211641&r1=1211640&r2=1211641&view=diff
==============================================================================
--- pig/branches/branch-0.10/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/SkewedPartitioner.java
(original)
+++ pig/branches/branch-0.10/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/SkewedPartitioner.java
Wed Dec  7 21:21:08 2011
@@ -112,7 +112,7 @@ public class SkewedPartitioner extends P
         try {
             Integer [] redCnt = new Integer[1]; 
             reducerMap = MapRedUtil.loadPartitionFileFromLocalCache(
-                    keyDistFile, redCnt, DataType.TUPLE);
+                    keyDistFile, redCnt, DataType.TUPLE, job);
             // check if the partition file is empty
             totalReducers = (redCnt[0] == null) ? -1 : redCnt[0];
         } catch (Exception e) {

Modified: pig/branches/branch-0.10/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartitionRearrange.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartitionRearrange.java?rev=1211641&r1=1211640&r2=1211641&view=diff
==============================================================================
--- pig/branches/branch-0.10/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartitionRearrange.java
(original)
+++ pig/branches/branch-0.10/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartitionRearrange.java
Wed Dec  7 21:21:08 2011
@@ -101,7 +101,7 @@ public class POPartitionRearrange extend
             Integer [] redCnt = new Integer[1];
 
             reducerMap = MapRedUtil.loadPartitionFileFromLocalCache(
-                    keyDistFile, redCnt, DataType.NULL);
+                    keyDistFile, redCnt, DataType.NULL, PigMapReduce.sJobConfInternal.get());
 
             // check if the partition file is empty
             totalReducers = (redCnt[0] == null) ? -1 : redCnt[0];

Modified: pig/branches/branch-0.10/src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java?rev=1211641&r1=1211640&r2=1211641&view=diff
==============================================================================
--- pig/branches/branch-0.10/src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java
(original)
+++ pig/branches/branch-0.10/src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java
Wed Dec  7 21:21:08 2011
@@ -78,7 +78,7 @@ public class MapRedUtil {
      */
     @SuppressWarnings("unchecked")
     public static <E> Map<E, Pair<Integer, Integer>> loadPartitionFileFromLocalCache(
-            String keyDistFile, Integer[] totalReducers, byte keyType)
+            String keyDistFile, Integer[] totalReducers, byte keyType, Configuration mapConf)
             throws IOException {
 
         Map<E, Pair<Integer, Integer>> reducerMap = new HashMap<E, Pair<Integer,
Integer>>();
@@ -86,6 +86,10 @@ public class MapRedUtil {
         // use local file system to get the keyDistFile
         Configuration conf = new Configuration(false);            
         
+        if (mapConf.get("yarn.resourcemanager.principal")!=null) {
+            conf.set("yarn.resourcemanager.principal", mapConf.get("yarn.resourcemanager.principal"));
+        }
+        
         if (PigMapReduce.sJobConfInternal.get().get("fs.file.impl")!=null)
             conf.set("fs.file.impl", PigMapReduce.sJobConfInternal.get().get("fs.file.impl"));
         if (PigMapReduce.sJobConfInternal.get().get("fs.hdfs.impl")!=null)



Mime
View raw message