hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r674628 - in /hadoop/core/branches/branch-0.18: CHANGES.txt src/test/org/apache/hadoop/mapred/SortValidator.java
Date Mon, 07 Jul 2008 20:44:41 GMT
Author: cdouglas
Date: Mon Jul  7 13:44:41 2008
New Revision: 674628

URL: http://svn.apache.org/viewvc?rev=674628&view=rev
Log:
HADOOP-3597. Fix SortValidator to use filesystems other than the default as
input. Validation job still runs on default fs. Contributed by Jothi
Padmanabhan.

Modified:
    hadoop/core/branches/branch-0.18/CHANGES.txt
    hadoop/core/branches/branch-0.18/src/test/org/apache/hadoop/mapred/SortValidator.java

Modified: hadoop/core/branches/branch-0.18/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.18/CHANGES.txt?rev=674628&r1=674627&r2=674628&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.18/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.18/CHANGES.txt Mon Jul  7 13:44:41 2008
@@ -716,6 +716,10 @@
     HADOOP-3683. Fix dfs metrics to count file listings rather than files
     listed. (lohit vijayarenu via cdouglas)
 
+    HADOOP-3597. Fix SortValidator to use filesystems other than the default as
+    input. Validation job still runs on default fs.
+    (Jothi Padmanabhan via cdouglas)
+
 Release 0.17.1 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/core/branches/branch-0.18/src/test/org/apache/hadoop/mapred/SortValidator.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.18/src/test/org/apache/hadoop/mapred/SortValidator.java?rev=674628&r1=674627&r2=674628&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.18/src/test/org/apache/hadoop/mapred/SortValidator.java
(original)
+++ hadoop/core/branches/branch-0.18/src/test/org/apache/hadoop/mapred/SortValidator.java
Mon Jul  7 13:44:41 2008
@@ -314,14 +314,16 @@
     
     static void checkRecords(Configuration defaults, 
                              Path sortInput, Path sortOutput) throws IOException {
-      FileSystem fs = FileSystem.get(defaults);
+      FileSystem inputfs = sortInput.getFileSystem(defaults);
+      FileSystem outputfs = sortOutput.getFileSystem(defaults);
+      FileSystem defaultfs = FileSystem.get(defaults);
       JobConf jobConf = new JobConf(defaults, RecordStatsChecker.class);
       jobConf.setJobName("sortvalidate-recordstats-checker");
 
       int noSortReduceTasks = 
-        fs.listStatus(sortOutput, sortPathsFilter).length;
+        outputfs.listStatus(sortOutput, sortPathsFilter).length;
       jobConf.setInt("sortvalidate.sort.reduce.tasks", noSortReduceTasks);
-      int noSortInputpaths = fs.listStatus(sortInput).length;
+      int noSortInputpaths =  inputfs.listStatus(sortInput).length;
 
       jobConf.setInputFormat(NonSplitableSequenceFileInputFormat.class);
       jobConf.setOutputFormat(SequenceFileOutputFormat.class);
@@ -339,8 +341,8 @@
       FileInputFormat.setInputPaths(jobConf, sortInput);
       FileInputFormat.addInputPath(jobConf, sortOutput);
       Path outputPath = new Path("/tmp/sortvalidate/recordstatschecker");
-      if (fs.exists(outputPath)) {
-        fs.delete(outputPath, true);
+      if (defaultfs.exists(outputPath)) {
+        defaultfs.delete(outputPath, true);
       }
       FileOutputFormat.setOutputPath(jobConf, outputPath);
       
@@ -365,7 +367,7 @@
       
       // Check to ensure that the statistics of the 
       // framework's sort-input and sort-output match
-      SequenceFile.Reader stats = new SequenceFile.Reader(fs, 
+      SequenceFile.Reader stats = new SequenceFile.Reader(defaultfs,
                                                           new Path(outputPath, "part-00000"),
defaults);
       IntWritable k1 = new IntWritable();
       IntWritable k2 = new IntWritable();



Mime
View raw message