hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1077804 - in /hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred: MapTask.java ReduceTask.java Task.java
Date Fri, 04 Mar 2011 05:24:01 GMT
Author: omalley
Date: Fri Mar  4 05:24:01 2011
New Revision: 1077804

URL: http://svn.apache.org/viewvc?rev=1077804&view=rev
Log:
commit 4713a5bf905e0b566a3ac7882f0e79bbb0f8fa7f
Author: Siddharth Seth <sseth@yahoo-inc.com>
Date:   Thu Feb 17 12:33:14 2011 -0800

     Qualify the paths before figuring out which filesystem
    statistics need to be tracked

Modified:
    hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/MapTask.java
    hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
    hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/Task.java

Modified: hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/MapTask.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/MapTask.java?rev=1077804&r1=1077803&r2=1077804&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/MapTask.java
(original)
+++ hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/MapTask.java
Fri Mar  4 05:24:01 2011
@@ -189,7 +189,7 @@ class MapTask extends Task {
 
       Statistics matchedStats = null;
       if (split instanceof FileSplit) {
-        matchedStats = getFsStatistics(((FileSplit) split).getPath());
+        matchedStats = getFsStatistics(((FileSplit) split).getPath(), job);
       } 
       fsStats = matchedStats;
       
@@ -480,7 +480,7 @@ class MapTask extends Task {
       Statistics matchedStats = null;
       if (split instanceof org.apache.hadoop.mapreduce.lib.input.FileSplit) {
         matchedStats = getFsStatistics(((org.apache.hadoop.mapreduce.lib.input.FileSplit)
split)
-            .getPath());
+            .getPath(), job);
       } 
       fsStats = matchedStats;
 	  
@@ -615,7 +615,7 @@ class MapTask extends Task {
       Statistics matchedStats = null;
       if (outputFormat instanceof org.apache.hadoop.mapreduce.lib.output.FileOutputFormat)
{
         matchedStats = getFsStatistics(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat
-            .getOutputPath(jobContext));
+            .getOutputPath(jobContext), job);
       }
       fsStats = matchedStats;
       mapOutputRecordCounter = 
@@ -808,7 +808,7 @@ class MapTask extends Task {
       
       Statistics matchedStats = null;
       if (outputFormat instanceof FileOutputFormat) {
-        matchedStats = getFsStatistics(FileOutputFormat.getOutputPath(job));
+        matchedStats = getFsStatistics(FileOutputFormat.getOutputPath(job), job);
       } 
       fsStats = matchedStats;
       mapOutputRecordCounter = reporter.getCounter(MAP_OUTPUT_RECORDS);

Modified: hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/ReduceTask.java?rev=1077804&r1=1077803&r2=1077804&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
(original)
+++ hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
Fri Mar  4 05:24:01 2011
@@ -438,7 +438,7 @@ class ReduceTask extends Task {
           .getCounter(FileOutputFormat.Counter.BYTES_WRITTEN);
       Statistics matchedStats = null;
       if (job.getOutputFormat() instanceof FileOutputFormat) {
-        matchedStats = getFsStatistics(FileOutputFormat.getOutputPath(job));
+        matchedStats = getFsStatistics(FileOutputFormat.getOutputPath(job), job);
       }
       fsStats = matchedStats;
 
@@ -560,7 +560,7 @@ class ReduceTask extends Task {
       // getTaskID());
       if (outputFormat instanceof org.apache.hadoop.mapreduce.lib.output.FileOutputFormat)
{
         matchedStats = getFsStatistics(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat
-            .getOutputPath(taskContext));
+            .getOutputPath(taskContext), taskContext.getConfiguration());
       }
       fsStats = matchedStats;
 

Modified: hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/Task.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/Task.java?rev=1077804&r1=1077803&r2=1077804&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/Task.java
(original)
+++ hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/Task.java
Fri Mar  4 05:24:01 2011
@@ -1021,8 +1021,10 @@ abstract public class Task implements Wr
    *          the path.
    * @return a Statistics instance, or null if none is found for the scheme.
    */
-  protected static Statistics getFsStatistics(Path path) {
+  protected static Statistics getFsStatistics(Path path, Configuration conf)
+      throws IOException {
     Statistics matchedStats = null;
+    path = path.getFileSystem(conf).makeQualified(path);
     for (Statistics stats : FileSystem.getAllStatistics()) {
       if (stats.getScheme().equals(path.toUri().getScheme())) {
         matchedStats = stats;



Mime
View raw message