hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r388306 - in /lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred: JobClient.java MapFileOutputFormat.java OutputFormat.java OutputFormatBase.java SequenceFileOutputFormat.java TextOutputFormat.java
Date Fri, 24 Mar 2006 00:28:01 GMT
Author: cutting
Date: Thu Mar 23 16:28:01 2006
New Revision: 388306

URL: http://svn.apache.org/viewcvs?rev=388306&view=rev
Log:
Move checking of output directory existence from JobClient to OutputFormat, so that it can
be overridden.  Add a base class for OutputFormat that implements this new method.

Added:
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormatBase.java
Modified:
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormat.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TextOutputFormat.java

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java?rev=388306&r1=388305&r2=388306&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java Thu Mar 23 16:28:01
2006
@@ -256,15 +256,8 @@
           job.setWorkingDirectory(fileSys.getWorkingDirectory().toString());          
         }
 
-        // Ensure that the output directory is set and not already there
-        File outDir = job.getOutputDir();
-        if (outDir == null && job.getNumReduceTasks() != 0) {
-            throw new IOException("Output directory not set in JobConf.");
-        }
-        if (outDir != null && fs.exists(outDir)) {
-            throw new IOException("Output directory " + outDir + 
-                                  " already exists.");
-        }
+        // Check the output specification
+        job.getOutputFormat().checkOutputSpecs(fs, job);
 
         // Write job file to JobTracker's fs        
         FSDataOutputStream out = fileSys.create(submitJobFile);

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java?rev=388306&r1=388305&r2=388306&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java Thu Mar
23 16:28:01 2006
@@ -28,7 +28,7 @@
 import org.apache.hadoop.conf.Configuration;
 
 /** An {@link OutputFormat} that writes {@link MapFile}s. */
-public class MapFileOutputFormat implements OutputFormat {
+public class MapFileOutputFormat extends OutputFormatBase {
 
   public RecordWriter getRecordWriter(FileSystem fs, JobConf job,
                                       String name) throws IOException {

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormat.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormat.java?rev=388306&r1=388305&r2=388306&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormat.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormat.java Thu Mar 23 16:28:01
2006
@@ -33,5 +33,15 @@
    */
   RecordWriter getRecordWriter(FileSystem fs, JobConf job, String name)
     throws IOException;
+
+  /** Check whether the output specification for a job is appropriate.  Called
+   * when a job is submitted.  Typically checks that it does not already exist,
+   * throwing an exception when it already exists, so that output is not
+   * overwritten.
+   *
+   * @param job the job whose output will be written
+   * @throws IOException when output should not be attempted
+   */
+  void checkOutputSpecs(FileSystem fs, JobConf job) throws IOException;
 }
 

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormatBase.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormatBase.java?rev=388306&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormatBase.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormatBase.java Thu Mar 23
16:28:01 2006
@@ -0,0 +1,43 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+import java.io.File;
+
+import org.apache.hadoop.fs.FileSystem;
+
+/** A base class for {@link OutputFormat}. */
+public abstract class OutputFormatBase implements OutputFormat {
+  public abstract RecordWriter getRecordWriter(FileSystem fs,
+                                               JobConf job, String name)
+    throws IOException;
+
+  public void checkOutputSpecs(FileSystem fs, JobConf job) throws IOException {
+    // Ensure that the output directory is set and not already there
+    File outDir = job.getOutputDir();
+    if (outDir == null && job.getNumReduceTasks() != 0) {
+      throw new IOException("Output directory not set in JobConf.");
+    }
+    if (outDir != null && fs.exists(outDir)) {
+      throw new IOException("Output directory " + outDir + 
+                            " already exists.");
+    }
+  }
+
+}
+

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java?rev=388306&r1=388305&r2=388306&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java Thu
Mar 23 16:28:01 2006
@@ -28,7 +28,7 @@
 import org.apache.hadoop.conf.Configuration;
 
 /** An {@link OutputFormat} that writes {@link SequenceFile}s. */
-public class SequenceFileOutputFormat implements OutputFormat {
+public class SequenceFileOutputFormat extends OutputFormatBase {
 
   public RecordWriter getRecordWriter(FileSystem fs, JobConf job,
                                       String name) throws IOException {

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TextOutputFormat.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TextOutputFormat.java?rev=388306&r1=388305&r2=388306&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TextOutputFormat.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TextOutputFormat.java Thu Mar 23
16:28:01 2006
@@ -26,7 +26,7 @@
 import org.apache.hadoop.io.Writable;
 
 /** An {@link OutputFormat} that writes plain text files. */
-public class TextOutputFormat implements OutputFormat {
+public class TextOutputFormat extends OutputFormatBase {
 
   public RecordWriter getRecordWriter(FileSystem fs, JobConf job,
                                       String name) throws IOException {



Mime
View raw message