hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r489185 - in /lucene/hadoop/trunk: ./ src/contrib/smallJobsBenchmark/src/java/org/apache/hadoop/benchmarks/mapred/ src/contrib/streaming/src/java/org/apache/hadoop/streaming/ src/java/org/apache/hadoop/conf/ src/java/org/apache/hadoop/dfs/ ...
Date Wed, 20 Dec 2006 19:53:03 GMT
Author: cutting
Date: Wed Dec 20 11:53:02 2006
New Revision: 489185

URL: http://svn.apache.org/viewvc?view=rev&rev=489185
Log:
HADOOP-836.  Fix a MapReduce bug on Windows where the wrong FileSystem was used.  Contributed
by Owen.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/src/java/org/apache/hadoop/benchmarks/mapred/MultiJobRunner.java
    lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/filecache/DistributedCache.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FsShell.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalFileSystem.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/IsolationRunner.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapTask.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestPread.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestSeekBug.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/DFSCIOTest.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystem.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestSequenceFileInputFilter.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestSequenceFileInputFormat.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestTextInputFormat.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/TestWritable.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Wed Dec 20 11:53:02 2006
@@ -137,6 +137,11 @@
 38. HADOOP-835.  Fix a NullPointerException reading record-compressed
     SequenceFiles.  (Hairong Kuang via cutting)
 
+39. HADOOP-836.  Fix a MapReduce bug on Windows, where the wrong
+    FileSystem was used.  Also add a static FileSystem.getLocal()
+    method and better Path checking in HDFS, to help avoid such issues
+    in the future.  (omalley via cutting)
+
 
 Release 0.9.2 - 2006-12-15
 

Modified: lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/src/java/org/apache/hadoop/benchmarks/mapred/MultiJobRunner.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/src/java/org/apache/hadoop/benchmarks/mapred/MultiJobRunner.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/src/java/org/apache/hadoop/benchmarks/mapred/MultiJobRunner.java
(original)
+++ lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/src/java/org/apache/hadoop/benchmarks/mapred/MultiJobRunner.java
Wed Dec 20 11:53:02 2006
@@ -231,7 +231,7 @@
     }
     //new File(localPath).
     Configuration conf = new Configuration();
-    FileSystem localFS = FileSystem.getNamed("local", conf);
+    FileSystem localFS = FileSystem.getLocal(conf);
     FileSystem remoteFS = FileSystem.get(conf);
     
     FileUtil.copy(localFS, new Path(localFile), remoteFS, 
@@ -249,7 +249,7 @@
   throws IOException{
     
     Configuration conf = new Configuration();
-    FileSystem localFS = FileSystem.getNamed("local", conf);
+    FileSystem localFS = FileSystem.getLocal(conf);
     FileSystem remoteFS = FileSystem.get(conf);
     
     FileUtil.copy(remoteFS, remotePath, 

Modified: lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
(original)
+++ lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
Wed Dec 20 11:53:02 2006
@@ -211,7 +211,7 @@
       fs_ = FileSystem.get(job_);
       if (job_.getBoolean("stream.sideoutput.localfs", false)) {
         //sideFs_ = new LocalFileSystem(job_);
-        sideFs_ = FileSystem.getNamed("local", job_);
+        sideFs_ = FileSystem.getLocal(job_);
       } else {
         sideFs_ = fs_;
       }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java Wed Dec 20 11:53:02
2006
@@ -363,7 +363,7 @@
     throws IOException {
     String[] dirs = getStrings(dirsProp);
     int hashCode = path.hashCode();
-    FileSystem fs = FileSystem.getNamed("local", this);
+    FileSystem fs = FileSystem.getLocal(this);
     for (int i = 0; i < dirs.length; i++) {  // try each local dir
       int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
       Path file = new Path(dirs[index], path);

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java Wed Dec
20 11:53:02 2006
@@ -96,12 +96,23 @@
     }
     
     public void setWorkingDirectory(Path dir) {
+      Path result = makeAbsolute(dir);
+      if (!FSNamesystem.isValidName(result.toString())) {
+        throw new IllegalArgumentException("Invalid DFS directory name " + 
+                                           result);
+      }
       workingDir = makeAbsolute(dir);
     }
     
     private UTF8 getPath(Path file) {
       checkPath(file);
-      return new UTF8(makeAbsolute(file).toUri().getPath());
+      String result = makeAbsolute(file).toUri().getPath();
+      if (!FSNamesystem.isValidName(result)) {
+        throw new IllegalArgumentException("Pathname " + result + " from " +
+                                            file +
+                                            " is not a valid DFS filename.");
+      }
+      return new UTF8(result);
     }
 
     public String[][] getFileCacheHints(Path f, long start, long len) throws IOException
{

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java Wed Dec 20 11:53:02
2006
@@ -845,7 +845,7 @@
      * Whether the pathname is valid.  Currently prohibits relative paths, 
      * and names which contain a ":" or "/" 
      */
-    private boolean isValidName(String src) {
+    static boolean isValidName(String src) {
       
       // Path must be absolute.
       if (!src.startsWith(Path.SEPARATOR)) {

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/filecache/DistributedCache.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/filecache/DistributedCache.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/filecache/DistributedCache.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/filecache/DistributedCache.java Wed Dec
20 11:53:02 2006
@@ -127,7 +127,7 @@
         CacheStatus lcacheStatus = (CacheStatus) cachedArchives.get(cacheId);
         if (lcacheStatus.refcount == 0) {
           // delete this cache entry
-          FileSystem.getNamed("local", conf).delete(lcacheStatus.localLoadPath);
+          FileSystem.getLocal(conf).delete(lcacheStatus.localLoadPath);
           it.remove();
         }
       }
@@ -195,7 +195,7 @@
         throw new IOException("Cache " + cacheStatus.localLoadPath.toString()
             + " is in use and cannot be refreshed");
       byte[] checkSum = createMD5(cache, conf);
-      FileSystem localFs = FileSystem.getNamed("local", conf);
+      FileSystem localFs = FileSystem.getLocal(conf);
       localFs.delete(cacheStatus.localLoadPath);
       Path parchive = new Path(cacheStatus.localLoadPath,
                                new Path(cacheStatus.localLoadPath.getName()));

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java Wed Dec 20 11:53:02
2006
@@ -118,7 +118,17 @@
 
       return get(URI.create(name), conf);
     }
-  
+
+    /**
+     * Get the local file syste
+     * @param conf the configuration to configure the file system with
+     * @return a LocalFileSystem
+     */
+    public static LocalFileSystem getLocal(Configuration conf)
+      throws IOException {
+      return (LocalFileSystem)get(LocalFileSystem.NAME, conf);
+    }
+
     /** Returns the FileSystem for this URI's scheme and authority.  The scheme
      * of the URI determines a configuration property name,
      * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem
class.

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FsShell.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FsShell.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FsShell.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FsShell.java Wed Dec 20 11:53:02 2006
@@ -152,10 +152,10 @@
       for( int i=0; i<srcs.length; i++ ) {
         if(endline) {
             FileUtil.copyMerge(fs, srcs[i], 
-                    FileSystem.getNamed("local", conf), dst, false, conf, "\n");
+                    FileSystem.getLocal(conf), dst, false, conf, "\n");
         } else {
             FileUtil.copyMerge(fs, srcs[i], 
-                    FileSystem.getNamed("local", conf), dst, false, conf, null);
+                    FileSystem.getLocal(conf), dst, false, conf, null);
         }
       }
     }      

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalFileSystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalFileSystem.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalFileSystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalFileSystem.java Wed Dec 20 11:53:02
2006
@@ -32,7 +32,7 @@
  * @author Mike Cafarella
  *****************************************************************/
 public class LocalFileSystem extends FileSystem {
-    private static final URI NAME = URI.create("file:///");
+    static final URI NAME = URI.create("file:///");
 
     private Path workingDir =
       new Path(System.getProperty("user.dir"));

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/IsolationRunner.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/IsolationRunner.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/IsolationRunner.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/IsolationRunner.java Wed Dec 20
11:53:02 2006
@@ -139,7 +139,7 @@
     int partition = conf.getInt("mapred.task.partition", 0);
     
     // setup the local and user working directories
-    FileSystem local = FileSystem.getNamed("local", conf);
+    FileSystem local = FileSystem.getLocal(conf);
     File workDirName = new File(jobFilename.getParent(), "work");
     local.setWorkingDirectory(new Path(workDirName.toString()));
     FileSystem.get(conf).setWorkingDirectory(conf.getWorkingDirectory());

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java Wed Dec 20 11:53:02
2006
@@ -129,14 +129,14 @@
   public void deleteLocalFiles() throws IOException {
     String[] localDirs = getLocalDirs();
     for (int i = 0; i < localDirs.length; i++) {
-      FileSystem.getNamed("local", this).delete(new Path(localDirs[i]));
+      FileSystem.getLocal(this).delete(new Path(localDirs[i]));
     }
   }
 
   public void deleteLocalFiles(String subdir) throws IOException {
     String[] localDirs = getLocalDirs();
     for (int i = 0; i < localDirs.length; i++) {
-      FileSystem.getNamed("local", this).delete(new Path(localDirs[i], subdir));
+      FileSystem.getLocal(this).delete(new Path(localDirs[i], subdir));
     }
   }
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java Wed Dec 20 11:53:02
2006
@@ -75,7 +75,7 @@
         this.jobtracker = jobtracker;
         this.status = new JobStatus(jobid, 0.0f, 0.0f, JobStatus.PREP);
         this.startTime = System.currentTimeMillis();
-        this.localFs = (LocalFileSystem)FileSystem.getNamed("local", default_conf);
+        this.localFs = (LocalFileSystem)FileSystem.getLocal(default_conf);
 
         JobConf default_job_conf = new JobConf(default_conf);
         this.localJobFile = default_job_conf.getLocalPath(JobTracker.SUBDIR 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java Wed Dec 20 11:53:02
2006
@@ -69,7 +69,7 @@
       this.mapoutputFile.setConf(conf);
 
       this.localFile = new JobConf(conf).getLocalPath("localRunner/"+id+".xml");
-      this.localFs = FileSystem.getNamed("local", conf);
+      this.localFs = FileSystem.getLocal(conf);
 
       fs.copyToLocalFile(new Path(file), localFile);
       this.job = new JobConf(localFile);

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapTask.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapTask.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapTask.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapTask.java Wed Dec 20 11:53:02
2006
@@ -107,7 +107,7 @@
     super.localizeConfiguration(conf);
     Path localSplit = new Path(new Path(getJobFile()).getParent(), 
                                "split.dta");
-    DataOutputStream out = LocalFileSystem.get(conf).create(localSplit);
+    DataOutputStream out = FileSystem.getLocal(conf).create(localSplit);
     split.write(out);
     out.close();
     if (split instanceof FileSplit) {
@@ -248,7 +248,7 @@
       this.comparator = job.getOutputKeyComparator();
       this.keyClass = job.getMapOutputKeyClass();
       this.valClass = job.getMapOutputValueClass();
-      this.localFs = FileSystem.getNamed("local", job);
+      this.localFs = FileSystem.getLocal(job);
       this.codec = null;
       this.compressionType = CompressionType.NONE;
       if (job.getCompressMapOutput()) {

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java Wed Dec 20 11:53:02
2006
@@ -227,7 +227,7 @@
     Class valueClass = job.getMapOutputValueClass();
     Reducer reducer = (Reducer)ReflectionUtils.newInstance(
                                   job.getReducerClass(), job);
-    FileSystem lfs = FileSystem.getNamed("local", job);
+    FileSystem lfs = FileSystem.getLocal(job);
 
     copyPhase.complete();                         // copy is already complete
     

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java Wed Dec 20
11:53:02 2006
@@ -258,7 +258,7 @@
     super(task, tracker, conf);
     this.mapOutputFile = new MapOutputFile();
     this.mapOutputFile.setConf(conf);
-    localFileSys = FileSystem.getNamed("local", conf);
+    localFileSys = FileSystem.getLocal(conf);
 
     this.reduceTask = (ReduceTask)getTask();
     this.scheduledCopies = new ArrayList(100);

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java Wed Dec 20 11:53:02
2006
@@ -107,7 +107,7 @@
           DistributedCache.setLocalFiles(conf, stringifyPathArray(p));
         }
         Path localTaskFile = new Path(t.getJobFile());
-        FileSystem localFs = FileSystem.getNamed("local", conf);
+        FileSystem localFs = FileSystem.getLocal(conf);
         localFs.delete(localTaskFile);
         OutputStream out = localFs.create(localTaskFile);
         try {
@@ -367,4 +367,4 @@
     }
   }
   
-}
\ No newline at end of file
+}

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java Wed Dec 20 11:53:02
2006
@@ -287,7 +287,7 @@
                                   jobId + Path.SEPARATOR + "job.jar");
   
           String jobFile = t.getJobFile();
-          FileSystem localFs = FileSystem.getNamed("local", fConf);
+          FileSystem localFs = FileSystem.getLocal(fConf);
           // this will happen on a partial execution of localizeJob.
           // Sometimes the job.xml gets copied but copying job.jar
           // might throw out an exception
@@ -415,7 +415,7 @@
       server.setThreads(1, workerThreads);
       // let the jsp pages get to the task tracker, config, and other relevant
       // objects
-      FileSystem local = FileSystem.getNamed("local", conf);
+      FileSystem local = FileSystem.getLocal(conf);
       server.setAttribute("task.tracker", this);
       server.setAttribute("local.file.system", local);
       server.setAttribute("conf", conf);
@@ -955,7 +955,7 @@
             Path localTaskDir =
               new Path(this.defaultJobConf.getLocalPath(TaskTracker.getJobCacheSubdir()),

                 (task.getJobId() + Path.SEPARATOR + task.getTaskId()));
-           FileSystem localFs = FileSystem.getNamed("local", fConf);
+           FileSystem localFs = FileSystem.getLocal(fConf);
            if (!localFs.mkdirs(localTaskDir)) {
              throw new IOException("Mkdirs failed to create " + localTaskDir.toString());
            }

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestPread.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestPread.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestPread.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestPread.java Wed Dec 20 11:53:02
2006
@@ -129,7 +129,7 @@
    */
   public void testPreadLocalFS() throws IOException {
     Configuration conf = new Configuration();
-    FileSystem fileSys = FileSystem.getNamed("local", conf);
+    FileSystem fileSys = FileSystem.getLocal(conf);
     try {
       Path file1 = new Path("build/test/data", "preadtest.dat");
       writeFile(fileSys, file1);

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestSeekBug.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestSeekBug.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestSeekBug.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestSeekBug.java Wed Dec 20 11:53:02
2006
@@ -108,7 +108,7 @@
    */
   public void testSeekBugLocalFS() throws IOException {
     Configuration conf = new Configuration();
-    FileSystem fileSys = FileSystem.getNamed("local", conf);
+    FileSystem fileSys = FileSystem.getLocal(conf);
     try {
       Path file1 = new Path("build/test/data", "seektest.dat");
       writeFile(fileSys, file1);

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/DFSCIOTest.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/DFSCIOTest.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/DFSCIOTest.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/DFSCIOTest.java Wed Dec 20 11:53:02
2006
@@ -198,7 +198,7 @@
       totalSize *= MEGA;
       
       // create instance of local filesystem 
-      FileSystem localFS = FileSystem.getNamed("local", fsConfig);
+      FileSystem localFS = FileSystem.getLocal(fsConfig);
       
       try {
     	  	// native runtime
@@ -300,7 +300,7 @@
     	  totalSize *= MEGA;
       
       // create instance of local filesystem 
-      FileSystem localFS = FileSystem.getNamed("local", fsConfig);
+      FileSystem localFS = FileSystem.getLocal(fsConfig);
       
       try {
   	  	// native runtime

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystem.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystem.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystem.java Wed Dec 20
11:53:02 2006
@@ -44,7 +44,7 @@
    */
   public void testWorkingDirectory() throws IOException {
     Configuration conf = new Configuration();
-    FileSystem fileSys = FileSystem.getNamed("local", conf);
+    FileSystem fileSys = FileSystem.getLocal(conf);
     Path origDir = fileSys.getWorkingDirectory();
     Path subdir = new Path("build/test/data/work-dir/new subdir");
     try {

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestSequenceFileInputFilter.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestSequenceFileInputFilter.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestSequenceFileInputFilter.java
(original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestSequenceFileInputFilter.java
Wed Dec 20 11:53:02 2006
@@ -46,7 +46,7 @@
   static {
       job.setInputPath(inDir);
       try {
-        fs = FileSystem.getNamed( "local", conf);
+        fs = FileSystem.getLocal(conf);
     } catch (IOException e) {
         e.printStackTrace();
         throw new RuntimeException(e);

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestSequenceFileInputFormat.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestSequenceFileInputFormat.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestSequenceFileInputFormat.java
(original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestSequenceFileInputFormat.java
Wed Dec 20 11:53:02 2006
@@ -36,7 +36,7 @@
 
   public void testFormat() throws Exception {
     JobConf job = new JobConf(conf);
-    FileSystem fs = FileSystem.getNamed("local", conf);
+    FileSystem fs = FileSystem.getLocal(conf);
     Path dir = new Path(System.getProperty("test.build.data",".") + "/mapred");
     Path file = new Path(dir, "test.seq");
     

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestTextInputFormat.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestTextInputFormat.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestTextInputFormat.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestTextInputFormat.java Wed Dec
20 11:53:02 2006
@@ -38,7 +38,7 @@
   private static FileSystem localFs = null; 
   static {
     try {
-      localFs = FileSystem.getNamed("local", defaultConf);
+      localFs = FileSystem.getLocal(defaultConf);
     } catch (IOException e) {
       throw new RuntimeException("init failure", e);
     }

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/TestWritable.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/TestWritable.java?view=diff&rev=489185&r1=489184&r2=489185
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/TestWritable.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/TestWritable.java Wed Dec 20
11:53:02 2006
@@ -42,7 +42,7 @@
 
   public void testFormat() throws Exception {
     JobConf job = new JobConf(conf);
-    FileSystem fs = FileSystem.getNamed("local", conf);
+    FileSystem fs = FileSystem.getLocal(conf);
     Path dir = new Path(System.getProperty("test.build.data",".") + "/mapred");
     Path file = new Path(dir, "test.seq");
     



Mime
View raw message