hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r508597 - in /lucene/hadoop/trunk: CHANGES.txt src/test/org/apache/hadoop/mapred/MRCaching.java src/test/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java src/test/org/apache/hadoop/mapred/TestMiniMRLocalFS.java
Date Fri, 16 Feb 2007 21:44:20 GMT
Author: cutting
Date: Fri Feb 16 13:44:19 2007
New Revision: 508597

URL: http://svn.apache.org/viewvc?view=rev&rev=508597
Log:
HADOOP-1021.  Fix MRCaching-based unit tests on Windows.  Contributed by Nigel.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MRCaching.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRLocalFS.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=508597&r1=508596&r2=508597
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Fri Feb 16 13:44:19 2007
@@ -59,6 +59,9 @@
 17. HADOOP-803.  Reduce memory use by HDFS namenode, phase I.
     (Raghu Angadi via cutting)
 
+18. HADOOP-1021.  Fix MRCaching-based unit tests on Windows.
+    (Nigel Daley via cutting)
+
 
 Branch 0.11 (unreleased)
 

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MRCaching.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MRCaching.java?view=diff&rev=508597&r1=508596&r2=508597
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MRCaching.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MRCaching.java Fri Feb 16 13:44:19
2007
@@ -60,18 +60,17 @@
       try {
         Path[] localArchives = DistributedCache.getLocalCacheArchives(conf);
         Path[] localFiles = DistributedCache.getLocalCacheFiles(conf);
-        FileSystem fs = FileSystem.get(conf);
         // read the cached files (unzipped, unjarred and text)
         // and put it into a single file TEST_ROOT_DIR/test.txt
         String TEST_ROOT_DIR = jconf.get("test.build.data","/tmp");
-        Path file = new Path(TEST_ROOT_DIR);
+        Path file = new Path("file:///",TEST_ROOT_DIR);
+        FileSystem fs = FileSystem.getLocal(conf);
         if (!fs.mkdirs(file)) {
           throw new IOException("Mkdirs failed to create " + file.toString());
         }
         Path fileOut = new Path(file, "test.txt");
         fs.delete(fileOut);
         DataOutputStream out = fs.create(fileOut);
-
         for (int i = 0; i < localArchives.length; i++) {
           // read out the files from these archives
           File f = new File(localArchives[i].toString());
@@ -126,10 +125,11 @@
   }
 
   public static boolean launchMRCache(String indir,
-      String outdir, JobConf conf, String input)
+      String outdir, String cacheDir, JobConf conf, String input)
       throws IOException {
     String TEST_ROOT_DIR = new Path(System.getProperty("test.build.data","/tmp"))
       .toString().replace(' ', '+');
+    //if (TEST_ROOT_DIR.startsWith("C:")) TEST_ROOT_DIR = "/tmp";
     conf.set("test.build.data",TEST_ROOT_DIR);
     final Path inDir = new Path(indir);
     final Path outDir = new Path(outdir);
@@ -139,6 +139,7 @@
       throw new IOException("Mkdirs failed to create " + inDir.toString());
     }
     {
+System.out.println("HERE:"+inDir);
       DataOutputStream file = fs.create(new Path(inDir, "part-0"));
       file.writeBytes(input);
       file.close();
@@ -162,19 +163,28 @@
     Path txtPath = new Path(localPath, new Path("test.txt"));
     Path jarPath = new Path(localPath, new Path("test.jar"));
     Path zipPath = new Path(localPath, new Path("test.zip"));
-    Path cacheTest = new Path(TEST_ROOT_DIR + "/cachedir");
-    fs.delete(cacheTest);
-    if (!fs.mkdirs(cacheTest)) {
-      throw new IOException("Mkdirs failed to create " + cacheTest.toString());
-    }
-    fs.copyFromLocalFile(txtPath, cacheTest);
-    fs.copyFromLocalFile(jarPath, cacheTest);
-    fs.copyFromLocalFile(zipPath, cacheTest);
+    Path cachePath = new Path(cacheDir);
+    fs.delete(cachePath);
+    if (!fs.mkdirs(cachePath)) {
+      throw new IOException("Mkdirs failed to create " + cachePath.toString());
+    }
+    fs.copyFromLocalFile(txtPath, cachePath);
+    fs.copyFromLocalFile(jarPath, cachePath);
+    fs.copyFromLocalFile(zipPath, cachePath);
     // setting the cached archives to zip, jar and simple text files
     String fileSys = fs.getName();
-    String archive1 = "dfs://" + fileSys + TEST_ROOT_DIR + "/cachedir/test.jar";
-    String archive2 = "dfs://" + fileSys + TEST_ROOT_DIR + "/cachedir/test.zip";
-    String file1 = "dfs://" + fileSys + TEST_ROOT_DIR + "/cachedir/test.txt";
+    String archive1;
+    String archive2;
+    String file1;
+    if (fileSys.equals("local")) {
+      archive1 = "file://" + cachePath + "/test.jar";
+      archive2 = "file://" + cachePath + "/test.zip";
+      file1 = "file://" + cachePath + "/test.txt";
+    } else {
+      archive1 = "dfs://" + fileSys + cachePath + "/test.jar";
+      archive2 = "dfs://" + fileSys + cachePath + "/test.zip";
+      file1 = "dfs://" + fileSys + cachePath + "/test.txt";
+    }
     URI uri1 = null;
     URI uri2 = null;
     URI uri3 = null;
@@ -193,8 +203,8 @@
     // match the real string. check if there are 3 instances or not.
     Path result = new Path(TEST_ROOT_DIR + "/test.txt");
     {
-      BufferedReader file = new BufferedReader(new InputStreamReader(fs
-          .open(result)));
+      BufferedReader file = new BufferedReader(new InputStreamReader(
+          FileSystem.getLocal(conf).open(result)));
       String line = file.readLine();
       while (line != null) {
         if (!testStr.equals(line))

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java?view=diff&rev=508597&r1=508596&r2=508597
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java Fri Feb
16 13:44:19 2007
@@ -42,6 +42,7 @@
       // run the wordcount example with caching
       boolean ret = MRCaching.launchMRCache("/testing/wc/input",
                                             "/testing/wc/output",
+                                            "/cachedir",
                                             mr.createJobConf(),
                                             "The quick brown fox\nhas many silly\n"
                                                 + "red fox sox\n");

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRLocalFS.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRLocalFS.java?view=diff&rev=508597&r1=508596&r2=508597
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRLocalFS.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRLocalFS.java Fri Feb 16
13:44:19 2007
@@ -47,6 +47,7 @@
           JobConf job = mr.createJobConf();
           boolean ret = MRCaching.launchMRCache(TEST_ROOT_DIR + "/wc/input",
                                                 TEST_ROOT_DIR + "/wc/output", 
+                                                TEST_ROOT_DIR + "/cachedir",
                                                 job,
                                                 "The quick brown fox\nhas many silly\n"
                                                     + "red fox sox\n");



Mime
View raw message