incubator-giraph-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ach...@apache.org
Subject svn commit: r1306014 - in /incubator/giraph/trunk: ./ src/main/java/org/apache/giraph/ src/main/java/org/apache/giraph/examples/ src/main/java/org/apache/giraph/graph/ src/main/java/org/apache/giraph/utils/ src/test/java/org/apache/giraph/
Date Tue, 27 Mar 2012 20:49:40 GMT
Author: aching
Date: Tue Mar 27 20:49:39 2012
New Revision: 1306014

URL: http://svn.apache.org/viewvc?rev=1306014&view=rev
Log:
GIRAPH-144: GiraphJob should not extend Job (users should not be able
to call Job methods like waitForCompletion or setMapper..etc)
(aching).

Modified:
    incubator/giraph/trunk/CHANGELOG
    incubator/giraph/trunk/src/main/java/org/apache/giraph/GiraphRunner.java
    incubator/giraph/trunk/src/main/java/org/apache/giraph/examples/SimpleCheckpointVertex.java
    incubator/giraph/trunk/src/main/java/org/apache/giraph/examples/SimpleShortestPathsVertex.java
    incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/GiraphJob.java
    incubator/giraph/trunk/src/main/java/org/apache/giraph/utils/InternalVertexRunner.java
    incubator/giraph/trunk/src/test/java/org/apache/giraph/BspCase.java
    incubator/giraph/trunk/src/test/java/org/apache/giraph/TestJsonBase64Format.java

Modified: incubator/giraph/trunk/CHANGELOG
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/CHANGELOG?rev=1306014&r1=1306013&r2=1306014&view=diff
==============================================================================
--- incubator/giraph/trunk/CHANGELOG (original)
+++ incubator/giraph/trunk/CHANGELOG Tue Mar 27 20:49:39 2012
@@ -2,6 +2,10 @@ Giraph Change Log
 
 Release 0.2.0 - unreleased
 
+  GIRAPH-144: GiraphJob should not extend Job (users should not be
+  able to call Job methods like waitForCompletion or setMapper..etc)
+  (aching).
+
   GIRAPH-159: Case insensitive file/directory name matching will
   produce errors on M/R jar unpack (bfem via aching).
 

Modified: incubator/giraph/trunk/src/main/java/org/apache/giraph/GiraphRunner.java
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/GiraphRunner.java?rev=1306014&r1=1306013&r2=1306014&view=diff
==============================================================================
--- incubator/giraph/trunk/src/main/java/org/apache/giraph/GiraphRunner.java (original)
+++ incubator/giraph/trunk/src/main/java/org/apache/giraph/GiraphRunner.java Tue Mar 27 20:49:39
2012
@@ -121,7 +121,8 @@ public class GiraphRunner implements Too
     job.setVertexOutputFormatClass(Class.forName(cmd.getOptionValue("of")));
 
     if (cmd.hasOption("ip")) {
-      FileInputFormat.addInputPath(job, new Path(cmd.getOptionValue("ip")));
+      FileInputFormat.addInputPath(job.getInternalJob(),
+                                   new Path(cmd.getOptionValue("ip")));
     } else {
       if (LOG.isInfoEnabled()) {
         LOG.info("No input path specified. Ensure your InputFormat does" +
@@ -130,7 +131,8 @@ public class GiraphRunner implements Too
     }
 
     if (cmd.hasOption("op")) {
-      FileOutputFormat.setOutputPath(job, new Path(cmd.getOptionValue("op")));
+      FileOutputFormat.setOutputPath(job.getInternalJob(),
+                                     new Path(cmd.getOptionValue("op")));
     } else {
       if (LOG.isInfoEnabled()) {
         LOG.info("No output path specified. Ensure your OutputFormat does" +

Modified: incubator/giraph/trunk/src/main/java/org/apache/giraph/examples/SimpleCheckpointVertex.java
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/examples/SimpleCheckpointVertex.java?rev=1306014&r1=1306013&r2=1306014&view=diff
==============================================================================
--- incubator/giraph/trunk/src/main/java/org/apache/giraph/examples/SimpleCheckpointVertex.java
(original)
+++ incubator/giraph/trunk/src/main/java/org/apache/giraph/examples/SimpleCheckpointVertex.java
Tue Mar 27 20:49:39 2012
@@ -226,8 +226,8 @@ public class SimpleCheckpointVertex exte
     int maxWorkers = Integer.parseInt(cmd.getOptionValue('w'));
     bspJob.setWorkerConfiguration(minWorkers, maxWorkers, 100.0f);
 
-    FileOutputFormat.setOutputPath(bspJob,
-        new Path(cmd.getOptionValue('o')));
+    FileOutputFormat.setOutputPath(bspJob.getInternalJob(),
+                                   new Path(cmd.getOptionValue('o')));
     boolean verbose = false;
     if (cmd.hasOption('v')) {
       verbose = true;

Modified: incubator/giraph/trunk/src/main/java/org/apache/giraph/examples/SimpleShortestPathsVertex.java
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/examples/SimpleShortestPathsVertex.java?rev=1306014&r1=1306013&r2=1306014&view=diff
==============================================================================
--- incubator/giraph/trunk/src/main/java/org/apache/giraph/examples/SimpleShortestPathsVertex.java
(original)
+++ incubator/giraph/trunk/src/main/java/org/apache/giraph/examples/SimpleShortestPathsVertex.java
Tue Mar 27 20:49:39 2012
@@ -262,8 +262,10 @@ public class SimpleShortestPathsVertex e
         SimpleShortestPathsVertexInputFormat.class);
     job.setVertexOutputFormatClass(
         SimpleShortestPathsVertexOutputFormat.class);
-    FileInputFormat.addInputPath(job, new Path(argArray[0]));
-    FileOutputFormat.setOutputPath(job, new Path(argArray[1]));
+    FileInputFormat.addInputPath(job.getInternalJob(),
+                                 new Path(argArray[0]));
+    FileOutputFormat.setOutputPath(job.getInternalJob(),
+                                   new Path(argArray[1]));
     job.getConfiguration().setLong(SimpleShortestPathsVertex.SOURCE_ID,
         Long.parseLong(argArray[2]));
     job.setWorkerConfiguration(Integer.parseInt(argArray[3]),

Modified: incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/GiraphJob.java
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/GiraphJob.java?rev=1306014&r1=1306013&r2=1306014&view=diff
==============================================================================
--- incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/GiraphJob.java (original)
+++ incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/GiraphJob.java Tue Mar 27
20:49:39 2012
@@ -29,10 +29,11 @@ import org.apache.log4j.Logger;
 import java.io.IOException;
 
 /**
- * Limits the functions that can be called by the user.  Job is too flexible
- * for our needs.  For instance, our job should not have any reduce tasks.
+ * Generates an appropriate internal {@link Job} for using Giraph in Hadoop.
+ * Uses composition to avoid unwanted {@link Job} methods from exposure
+ * to the user.
  */
-public class GiraphJob extends Job {
+public class GiraphJob {
   static {
     Configuration.addDefaultResource("giraph-site.xml");
   }
@@ -345,6 +346,12 @@ public class GiraphJob extends Job {
   /** Class logger */
   private static final Logger LOG = Logger.getLogger(GiraphJob.class);
 
+  /** Internal job that actually is submitted */
+  private final Job job;
+  /** Helper configuration from the job */
+  private final Configuration conf;
+
+
   /**
    * Constructor that will instantiate the configuration
    *
@@ -352,7 +359,7 @@ public class GiraphJob extends Job {
    * @throws IOException
    */
   public GiraphJob(String jobName) throws IOException {
-    super(new Configuration(), jobName);
+    this(new Configuration(), jobName);
   }
 
   /**
@@ -363,10 +370,30 @@ public class GiraphJob extends Job {
    * @throws IOException
    */
   public GiraphJob(Configuration conf, String jobName) throws IOException {
-    super(conf, jobName);
+    job = new Job(conf, jobName);
+    this.conf = job.getConfiguration();
   }
 
   /**
+   * Get the configuration from the internal job.
+   *
+   * @return Configuration used by the job.
+   */
+  public Configuration getConfiguration() {
+    return conf;
+  }
+
+  /**
+   * Be very cautious when using this method as it returns the internal job
+   * of {@link GiraphJob}.  This should only be used for methods that require
+   * access to the actual {@link Job}, i.e. FileInputFormat#addInputPath().
+   *
+   * @return Internal job that will actually be submitted to Hadoop.
+   */
+  public Job getInternalJob() {
+    return job;
+  }
+  /**
    * Make sure the configuration is set properly by the user prior to
    * submitting the job.
    */
@@ -580,7 +607,7 @@ public class GiraphJob extends Job {
     throws IOException, InterruptedException, ClassNotFoundException {
     checkConfiguration();
     checkLocalJobRunnerConfiguration(conf);
-    setNumReduceTasks(0);
+    job.setNumReduceTasks(0);
     // Most users won't hit this hopefully and can set it higher if desired
     setIntConfIfDefault("mapreduce.job.counters.limit", 512);
 
@@ -596,16 +623,16 @@ public class GiraphJob extends Job {
     // (DEFAULT_PING_INTERVAL)
     Client.setPingInterval(conf, 60000 * 5);
 
-    if (getJar() == null) {
-      setJarByClass(GiraphJob.class);
+    if (job.getJar() == null) {
+      job.setJarByClass(GiraphJob.class);
     }
     // Should work in MAPREDUCE-1938 to let the user jars/classes
     // get loaded first
     conf.setBoolean("mapreduce.user.classpath.first", true);
 
-    setMapperClass(GraphMapper.class);
-    setInputFormatClass(BspInputFormat.class);
-    setOutputFormatClass(BspOutputFormat.class);
-    return waitForCompletion(verbose);
+    job.setMapperClass(GraphMapper.class);
+    job.setInputFormatClass(BspInputFormat.class);
+    job.setOutputFormatClass(BspOutputFormat.class);
+    return job.waitForCompletion(verbose);
   }
 }

Modified: incubator/giraph/trunk/src/main/java/org/apache/giraph/utils/InternalVertexRunner.java
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/utils/InternalVertexRunner.java?rev=1306014&r1=1306013&r2=1306014&view=diff
==============================================================================
--- incubator/giraph/trunk/src/main/java/org/apache/giraph/utils/InternalVertexRunner.java
(original)
+++ incubator/giraph/trunk/src/main/java/org/apache/giraph/utils/InternalVertexRunner.java
Tue Mar 27 20:49:39 2012
@@ -129,8 +129,10 @@ public class InternalVertexRunner {
         conf.set(param.getKey(), param.getValue());
       }
 
-      FileInputFormat.addInputPath(job, new Path(inputFile.toString()));
-      FileOutputFormat.setOutputPath(job, new Path(outputDir.toString()));
+      FileInputFormat.addInputPath(job.getInternalJob(),
+                                   new Path(inputFile.toString()));
+      FileOutputFormat.setOutputPath(job.getInternalJob(),
+                                     new Path(outputDir.toString()));
 
       // configure a local zookeeper instance
       Properties zkProperties = new Properties();

Modified: incubator/giraph/trunk/src/test/java/org/apache/giraph/BspCase.java
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/test/java/org/apache/giraph/BspCase.java?rev=1306014&r1=1306013&r2=1306014&view=diff
==============================================================================
--- incubator/giraph/trunk/src/test/java/org/apache/giraph/BspCase.java (original)
+++ incubator/giraph/trunk/src/test/java/org/apache/giraph/BspCase.java Tue Mar 27 20:49:39
2012
@@ -26,7 +26,6 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.zookeeper.WatchedEvent;
 import org.apache.zookeeper.Watcher;
@@ -41,215 +40,213 @@ import junit.framework.TestCase;
  * Extended TestCase for making setting up Bsp testing.
  */
 public class BspCase extends TestCase implements Watcher {
-    /** JobTracker system property */
-    private final String jobTracker =
-        System.getProperty("prop.mapred.job.tracker");
-    /** Jar location system property */
-    private final String jarLocation =
-        System.getProperty("prop.jarLocation", "");
-    /** Number of actual processes for the BSP application */
-    private int numWorkers = 1;
-    /** ZooKeeper list system property */
-    private final String zkList = System.getProperty("prop.zookeeper.list");
-
-    /**
-     * Adjust the configuration to the basic test case
-     */
-    public final void setupConfiguration(GiraphJob job) {
-        Configuration conf = job.getConfiguration();
-        conf.set("mapred.jar", getJarLocation());
-
-        // Allow this test to be run on a real Hadoop setup
-        if (getJobTracker() != null) {
-            System.out.println("setup: Sending job to job tracker " +
-                       getJobTracker() + " with jar path " + getJarLocation()
-                       + " for " + getName());
-            conf.set("mapred.job.tracker", getJobTracker());
-            job.setWorkerConfiguration(getNumWorkers(),
-                                       getNumWorkers(),
-                                       100.0f);
-        }
-        else {
-            System.out.println("setup: Using local job runner with " +
-                               "location " + getJarLocation() + " for "
-                               + getName());
-            job.setWorkerConfiguration(1, 1, 100.0f);
-            // Single node testing
-            conf.setBoolean(GiraphJob.SPLIT_MASTER_WORKER, false);
-        }
-        conf.setInt(GiraphJob.POLL_ATTEMPTS, 10);
-        conf.setInt(GiraphJob.POLL_MSECS, 3*1000);
-        conf.setInt(GiraphJob.ZOOKEEPER_SERVERLIST_POLL_MSECS, 500);
-        if (getZooKeeperList() != null) {
-            job.setZooKeeperConfiguration(getZooKeeperList());
-        }
-        // GeneratedInputSplit will generate 5 vertices
-        conf.setLong(GeneratedVertexReader.READER_VERTICES, 5);
-    }
-
-    /**
-     * Create the test case
-     *
-     * @param testName name of the test case
-     */
-    public BspCase(String testName) {
-        super(testName);
-
-    }
-
-    /**
-     * Get the number of workers used in the BSP application
-     *
-     * @param numProcs number of processes to use
-     */
-    public int getNumWorkers() {
-        return numWorkers;
-    }
-
-    /**
-     * Get the ZooKeeper list
-     */
-    public String getZooKeeperList() {
-        return zkList;
-    }
-
-    /**
-     * Get the jar location
-     *
-     * @return location of the jar file
-     */
-    String getJarLocation() {
-        return jarLocation;
-    }
-
-    /**
-     * Get the job tracker location
-     *
-     * @return job tracker location as a string
-     */
-    String getJobTracker() {
-        return jobTracker;
-    }
-
-    /**
-     * Get the single part file status and make sure there is only one part
-     *
-     * @param fs Filesystem to look for the part file
-     * @param partDirPath Directory where the single part file should exist
-     * @return Single part file status
-     * @throws IOException
-     */
-    public static FileStatus getSinglePartFileStatus(Job job,
-                                                     Path partDirPath)
-            throws IOException {
-        FileSystem fs = FileSystem.get(job.getConfiguration());
-        FileStatus[] statusArray = fs.listStatus(partDirPath);
-        FileStatus singlePartFileStatus = null;
-        int partFiles = 0;
-        for (FileStatus fileStatus : statusArray) {
-            if (fileStatus.getPath().getName().equals("part-m-00000")) {
-                singlePartFileStatus = fileStatus;
+  /** JobTracker system property */
+  private final String jobTracker =
+      System.getProperty("prop.mapred.job.tracker");
+  /** Jar location system property */
+  private final String jarLocation =
+      System.getProperty("prop.jarLocation", "");
+  /** Number of actual processes for the BSP application */
+  private int numWorkers = 1;
+  /** ZooKeeper list system property */
+  private final String zkList = System.getProperty("prop.zookeeper.list");
+
+  /**
+   * Adjust the configuration to the basic test case
+   */
+  public final void setupConfiguration(GiraphJob job) {
+    Configuration conf = job.getConfiguration();
+    conf.set("mapred.jar", getJarLocation());
+
+    // Allow this test to be run on a real Hadoop setup
+    if (getJobTracker() != null) {
+      System.out.println("setup: Sending job to job tracker " +
+          getJobTracker() + " with jar path " + getJarLocation()
+          + " for " + getName());
+      conf.set("mapred.job.tracker", getJobTracker());
+      job.setWorkerConfiguration(getNumWorkers(),
+          getNumWorkers(),
+          100.0f);
+    }
+    else {
+      System.out.println("setup: Using local job runner with " +
+          "location " + getJarLocation() + " for "
+          + getName());
+      job.setWorkerConfiguration(1, 1, 100.0f);
+      // Single node testing
+      conf.setBoolean(GiraphJob.SPLIT_MASTER_WORKER, false);
+    }
+    conf.setInt(GiraphJob.POLL_ATTEMPTS, 10);
+    conf.setInt(GiraphJob.POLL_MSECS, 3*1000);
+    conf.setInt(GiraphJob.ZOOKEEPER_SERVERLIST_POLL_MSECS, 500);
+    if (getZooKeeperList() != null) {
+      job.setZooKeeperConfiguration(getZooKeeperList());
+    }
+    // GeneratedInputSplit will generate 5 vertices
+    conf.setLong(GeneratedVertexReader.READER_VERTICES, 5);
+  }
+
+  /**
+   * Create the test case
+   *
+   * @param testName name of the test case
+   */
+  public BspCase(String testName) {
+    super(testName);
+
+  }
+
+  /**
+   * Get the number of workers used in the BSP application
+   *
+   * @param numProcs number of processes to use
+   */
+  public int getNumWorkers() {
+    return numWorkers;
+  }
+
+  /**
+   * Get the ZooKeeper list
+   */
+  public String getZooKeeperList() {
+    return zkList;
+  }
+
+  /**
+   * Get the jar location
+   *
+   * @return location of the jar file
+   */
+  String getJarLocation() {
+    return jarLocation;
+  }
+
+  /**
+   * Get the job tracker location
+   *
+   * @return job tracker location as a string
+   */
+  String getJobTracker() {
+    return jobTracker;
+  }
+
+  /**
+   * Get the single part file status and make sure there is only one part
+   *
+   * @param job Job to get the file system from
+   * @param partDirPath Directory where the single part file should exist
+   * @return Single part file status
+   * @throws IOException
+   */
+  public static FileStatus getSinglePartFileStatus(GiraphJob job,
+      Path partDirPath) throws IOException {
+    FileSystem fs = FileSystem.get(job.getConfiguration());
+    FileStatus[] statusArray = fs.listStatus(partDirPath);
+    FileStatus singlePartFileStatus = null;
+    int partFiles = 0;
+    for (FileStatus fileStatus : statusArray) {
+      if (fileStatus.getPath().getName().equals("part-m-00000")) {
+        singlePartFileStatus = fileStatus;
+      }
+      if (fileStatus.getPath().getName().startsWith("part-m-")) {
+        ++partFiles;
+      }
+    }
+    if (partFiles != 1) {
+      throw new ArithmeticException(
+          "getSinglePartFile: Part file count should be 1, but is " +
+              partFiles);
+    }
+    return singlePartFileStatus;
+  }
+
+  @Override
+  public void setUp() {
+    if (jobTracker != null) {
+      System.out.println("Setting tasks to 3 for " + getName() +
+          " since JobTracker exists...");
+      numWorkers = 3;
+    }
+    try {
+      Configuration conf = new Configuration();
+      FileSystem hdfs = FileSystem.get(conf);
+      // Since local jobs always use the same paths, remove them
+      Path oldLocalJobPaths = new Path(
+          GiraphJob.ZOOKEEPER_MANAGER_DIR_DEFAULT);
+      FileStatus[] fileStatusArr;
+      try {
+        fileStatusArr = hdfs.listStatus(oldLocalJobPaths);
+        for (FileStatus fileStatus : fileStatusArr) {
+          if (fileStatus.isDir() &&
+              fileStatus.getPath().getName().contains("job_local")) {
+            System.out.println("Cleaning up local job path " +
+                fileStatus.getPath().getName());
+            hdfs.delete(oldLocalJobPaths, true);
+          }
+        }
+      } catch (FileNotFoundException e) {
+        // ignore this FileNotFound exception and continue.
+      }
+      if (zkList == null) {
+        return;
+      }
+      ZooKeeperExt zooKeeperExt =
+          new ZooKeeperExt(zkList, 30*1000, this);
+      List<String> rootChildren = zooKeeperExt.getChildren("/", false);
+      for (String rootChild : rootChildren) {
+        if (rootChild.startsWith("_hadoopBsp")) {
+          List<String> children =
+              zooKeeperExt.getChildren("/" + rootChild, false);
+          for (String child: children) {
+            if (child.contains("job_local_")) {
+              System.out.println("Cleaning up /_hadoopBsp/" +
+                  child);
+              zooKeeperExt.deleteExt(
+                  "/_hadoopBsp/" + child, -1, true);
             }
-            if (fileStatus.getPath().getName().startsWith("part-m-")) {
-                ++partFiles;
-            }
-        }
-        if (partFiles != 1) {
-            throw new ArithmeticException(
-                "getSinglePartFile: Part file count should be 1, but is " +
-                partFiles);
-        }
-        return singlePartFileStatus;
-    }
-
-    @Override
-    public void setUp() {
-        if (jobTracker != null) {
-            System.out.println("Setting tasks to 3 for " + getName() +
-                               " since JobTracker exists...");
-            numWorkers = 3;
+          }
         }
-        try {
-            Configuration conf = new Configuration();
-            FileSystem hdfs = FileSystem.get(conf);
-            // Since local jobs always use the same paths, remove them
-            Path oldLocalJobPaths = new Path(
-                GiraphJob.ZOOKEEPER_MANAGER_DIR_DEFAULT);
-            FileStatus[] fileStatusArr;
-            try {
-                fileStatusArr = hdfs.listStatus(oldLocalJobPaths);
-                for (FileStatus fileStatus : fileStatusArr) {
-                    if (fileStatus.isDir() &&
-                        fileStatus.getPath().getName().contains("job_local")) {
-                        System.out.println("Cleaning up local job path " +
-                            fileStatus.getPath().getName());
-                        hdfs.delete(oldLocalJobPaths, true);
-                    }
-                }
-            } catch (FileNotFoundException e) {
-                // ignore this FileNotFound exception and continue.
-            }
-            if (zkList == null) {
-                return;
-            }
-            ZooKeeperExt zooKeeperExt =
-                new ZooKeeperExt(zkList, 30*1000, this);
-            List<String> rootChildren = zooKeeperExt.getChildren("/", false);
-            for (String rootChild : rootChildren) {
-                if (rootChild.startsWith("_hadoopBsp")) {
-                    List<String> children =
-                        zooKeeperExt.getChildren("/" + rootChild, false);
-                    for (String child: children) {
-                        if (child.contains("job_local_")) {
-                            System.out.println("Cleaning up /_hadoopBsp/" +
-                                               child);
-                            zooKeeperExt.deleteExt(
-                                "/_hadoopBsp/" + child, -1, true);
-                        }
-                    }
-                }
-            }
-            zooKeeperExt.close();
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    @Override
-    public void process(WatchedEvent event) {
-        // Do nothing
-    }
-
-    /**
-     * Helper method to remove an old output directory if it exists,
-     * and set the output path for any VertexOutputFormat that uses
-     * FileOutputFormat.
-     *
-     * @param job Job to set the output path for
-     * @param outputPathString Path to output as a string
-     * @throws IOException
-     */
-    public static void removeAndSetOutput(GiraphJob job,
-                                          Path outputPath)
-            throws IOException {
-        remove(job.getConfiguration(), outputPath);
-        FileOutputFormat.setOutputPath(job, outputPath);
-    }
-    
-    /**
-     * Helper method to remove a path if it exists.
-     * 
-     * @param conf Configuration to load FileSystem from
-     * @param path Path to remove
-     * @throws IOException
-     */
-    public static void remove(Configuration conf, Path path) 
-            throws IOException {
-        FileSystem hdfs = FileSystem.get(conf);
-        hdfs.delete(path, true);
-    }
-
-    public static String getCallingMethodName() {
-        return Thread.currentThread().getStackTrace()[2].getMethodName();
-    }
+      }
+      zooKeeperExt.close();
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public void process(WatchedEvent event) {
+    // Do nothing
+  }
+
+  /**
+   * Helper method to remove an old output directory if it exists,
+   * and set the output path for any VertexOutputFormat that uses
+   * FileOutputFormat.
+   *
+   * @param job Job to set the output path for
+   * @param outputPathString Path to output as a string
+   * @throws IOException
+   */
+  public static void removeAndSetOutput(GiraphJob job,
+      Path outputPath) throws IOException {
+    remove(job.getConfiguration(), outputPath);
+    FileOutputFormat.setOutputPath(job.getInternalJob(), outputPath);
+  }
+
+  /**
+   * Helper method to remove a path if it exists.
+   *
+   * @param conf Configuration to load FileSystem from
+   * @param path Path to remove
+   * @throws IOException
+   */
+  public static void remove(Configuration conf, Path path)
+      throws IOException {
+    FileSystem hdfs = FileSystem.get(conf);
+    hdfs.delete(path, true);
+  }
+
+  public static String getCallingMethodName() {
+    return Thread.currentThread().getStackTrace()[2].getMethodName();
+  }
 }

Modified: incubator/giraph/trunk/src/test/java/org/apache/giraph/TestJsonBase64Format.java
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/test/java/org/apache/giraph/TestJsonBase64Format.java?rev=1306014&r1=1306013&r2=1306014&view=diff
==============================================================================
--- incubator/giraph/trunk/src/test/java/org/apache/giraph/TestJsonBase64Format.java (original)
+++ incubator/giraph/trunk/src/test/java/org/apache/giraph/TestJsonBase64Format.java Tue Mar
27 20:49:39 2012
@@ -84,7 +84,7 @@ public class TestJsonBase64Format extend
     job.setVertexInputFormatClass(JsonBase64VertexInputFormat.class);
     job.setVertexOutputFormatClass(JsonBase64VertexOutputFormat.class);
     job.getConfiguration().setInt(PageRankComputation.SUPERSTEP_COUNT, 3);
-    FileInputFormat.setInputPaths(job, outputPath);
+    FileInputFormat.setInputPaths(job.getInternalJob(), outputPath);
     Path outputPath2 = new Path("/tmp/" + getCallingMethodName() + "2");
     removeAndSetOutput(job, outputPath2);
     assertTrue(job.run(true));



Mime
View raw message