avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From th...@apache.org
Subject svn commit: r1179946 - in /avro/trunk: ./ lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/ lang/java/mapred/src/test/java/org/apache/avro/mapred/tether/ lang/java/tools/src/main/java/org/apache/avro/tool/
Date Fri, 07 Oct 2011 07:20:50 GMT
Author: thiru
Date: Fri Oct  7 07:20:49 2011
New Revision: 1179946

URL: http://svn.apache.org/viewvc?rev=1179946&view=rev
Log:
AVRO-912. Mapreduce tether test fails on Windows

Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherJob.java
    avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetheredProcess.java
    avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/tether/TestWordCountTether.java
    avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/tether/WordCountTask.java
    avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TetherTool.java

Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1179946&r1=1179945&r2=1179946&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Fri Oct  7 07:20:49 2011
@@ -119,6 +119,8 @@ Avro 1.6.0 (unreleased)
 
     AVRO-900. On slower machines Java unit test TestFileSpanStorage fails. (thiru)
 
+    AVRO-912. Mapreduce tether test fails on Windows. (thiru)
+
 Avro 1.5.4 (12 September 2011)
 
   IMPROVEMENTS

Modified: avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherJob.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherJob.java?rev=1179946&r1=1179945&r2=1179946&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherJob.java
(original)
+++ avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherJob.java
Fri Oct  7 07:20:49 2011
@@ -18,8 +18,11 @@
 
 package org.apache.avro.mapred.tether;
 
+import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
 import java.net.URI;
 import java.net.URISyntaxException;
 
@@ -49,8 +52,8 @@ public class TetherJob extends Configure
   }
   
   /** Set the URI for the application's executable. Normally this in HDFS. */
-  public static void setExecutable(JobConf job, URI executable) {
-    setExecutable(job,executable,"",false);
+  public static void setExecutable(JobConf job, File executable) {
+    setExecutable(job,executable, new ArrayList<String>(),false);
   }
   
   /**
@@ -63,9 +66,14 @@ public class TetherJob extends Configure
    *               - if false its not cached. I.e if the file is already stored on each local
file system
    *                or if its on a NFS share
    */
-  public static void setExecutable(JobConf job, URI executable,String argstr,boolean cached)
{
+  public static void setExecutable(JobConf job, File executable, List<String> args,
boolean cached) {
         job.set(TETHER_EXEC, executable.toString());
-        job.set(TETHER_EXEC_ARGS, argstr);
+        StringBuilder sb = new StringBuilder();
+        for (String a : args) {
+          sb.append(a);
+          sb.append('\n');
+        }
+        job.set(TETHER_EXEC_ARGS, sb.toString());
         job.set(TETHER_EXEC_CACHED,  (new Boolean(cached)).toString());
   }
 

Modified: avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetheredProcess.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetheredProcess.java?rev=1179946&r1=1179945&r2=1179946&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetheredProcess.java
(original)
+++ avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetheredProcess.java
Fri Oct  7 07:20:49 2011
@@ -137,7 +137,7 @@ class TetheredProcess  {
     // if we pass a single string containing all arguments we get quoted
     // incorrectly
     String args=job.get(TetherJob.TETHER_EXEC_ARGS);
-    String[] aparams=args.split(" ");
+    String[] aparams=args.split("\n");
     for (int i=0;i<aparams.length; i++){            
       aparams[i]=aparams[i].trim();
       if (aparams[i].length()>0){
@@ -165,6 +165,7 @@ class TetheredProcess  {
 
     // start child process
     ProcessBuilder builder = new ProcessBuilder(command);
+    System.out.println(command);
     builder.environment().putAll(env);
     return builder.start();
   }

Modified: avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/tether/TestWordCountTether.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/tether/TestWordCountTether.java?rev=1179946&r1=1179945&r2=1179946&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/tether/TestWordCountTether.java
(original)
+++ avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/tether/TestWordCountTether.java
Fri Oct  7 07:20:49 2011
@@ -72,7 +72,7 @@ import org.apache.avro.file.DataFileStre
 public class TestWordCountTether {
 
 
-  @Test
+    @Test
     @SuppressWarnings("deprecation")
     public void testJob() throws Exception {
 
@@ -86,19 +86,21 @@ public class TestWordCountTether {
     // create the input file
     WordCountUtil.writeLinesFile();
 
-    java.net.URI exec =
-      new java.net.URI(System.getProperty("java.home")+"/bin/java");
+    File exec =
+      new File(System.getProperty("java.home")+"/bin/java");
 
     //input path
     String in=dir+"/in";
 
     //create a string of the arguments
-    String execargs="-classpath " + System.getProperty("java.class.path");
-    execargs+= " org.apache.avro.mapred.tether.WordCountTask";
+    List<String> execargs = new ArrayList<String>();
+    execargs.add("-classpath");
+    execargs.add(System.getProperty("java.class.path"));
+    execargs.add("org.apache.avro.mapred.tether.WordCountTask");
 
     FileInputFormat.addInputPaths(job, in);
     FileOutputFormat.setOutputPath(job, outputPath);
-    TetherJob.setExecutable(job, exec,execargs,false);
+    TetherJob.setExecutable(job, exec, execargs, false);
 
     Schema outscheme= new Pair<Utf8,Long>(new Utf8(""), 0L).getSchema();
     AvroJob.setInputSchema(job, Schema.create(Schema.Type.STRING));

Modified: avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/tether/WordCountTask.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/tether/WordCountTask.java?rev=1179946&r1=1179945&r2=1179946&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/tether/WordCountTask.java
(original)
+++ avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/tether/WordCountTask.java
Fri Oct  7 07:20:49 2011
@@ -21,8 +21,8 @@ package org.apache.avro.mapred.tether;
 import java.io.IOException;
 import java.util.StringTokenizer;
 
-import org.apache.avro.util.Utf8;
 import org.apache.avro.mapred.Pair;
+import org.apache.avro.util.Utf8;
 
 /** Example Java tethered mapreduce executable.  Implements map and reduce
  * functions for word count. */

Modified: avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TetherTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TetherTool.java?rev=1179946&r1=1179945&r2=1179946&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TetherTool.java (original)
+++ avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TetherTool.java Fri Oct
 7 07:20:49 2011
@@ -20,7 +20,6 @@ package org.apache.avro.tool;
 import java.io.File;
 import java.io.InputStream;
 import java.io.PrintStream;
-import java.net.URI;
 import java.util.List;
 
 import joptsimple.OptionParser;
@@ -54,9 +53,9 @@ public class TetherTool implements Tool 
       List<String> args) throws Exception {
 
     OptionParser p = new OptionParser();
-    OptionSpec<URI> exec = p
+    OptionSpec<File> exec = p
         .accepts("program", "executable program, usually in HDFS")
-        .withRequiredArg().ofType(URI.class);
+        .withRequiredArg().ofType(File.class);
     OptionSpec<String> in = p.accepts("in", "comma-separated input paths")
         .withRequiredArg().ofType(String.class);
     OptionSpec<Path> out = p.accepts("out", "output directory")



Mime
View raw message