From hadoop-commits-return-129-apmail-lucene-hadoop-commits-archive=lucene.apache.org@lucene.apache.org Fri Mar 24 00:28:25 2006 Return-Path: Delivered-To: apmail-lucene-hadoop-commits-archive@locus.apache.org Received: (qmail 38735 invoked from network); 24 Mar 2006 00:28:24 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (209.237.227.199) by minotaur.apache.org with SMTP; 24 Mar 2006 00:28:24 -0000 Received: (qmail 18508 invoked by uid 500); 24 Mar 2006 00:28:24 -0000 Delivered-To: apmail-lucene-hadoop-commits-archive@lucene.apache.org Received: (qmail 18463 invoked by uid 500); 24 Mar 2006 00:28:24 -0000 Mailing-List: contact hadoop-commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hadoop-dev@lucene.apache.org Delivered-To: mailing list hadoop-commits@lucene.apache.org Received: (qmail 18443 invoked by uid 99); 24 Mar 2006 00:28:24 -0000 Received: from asf.osuosl.org (HELO asf.osuosl.org) (140.211.166.49) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 23 Mar 2006 16:28:24 -0800 X-ASF-Spam-Status: No, hits=-9.4 required=10.0 tests=ALL_TRUSTED,NO_REAL_NAME X-Spam-Check-By: apache.org Received: from [209.237.227.194] (HELO minotaur.apache.org) (209.237.227.194) by apache.org (qpsmtpd/0.29) with SMTP; Thu, 23 Mar 2006 16:28:23 -0800 Received: (qmail 38644 invoked by uid 65534); 24 Mar 2006 00:28:03 -0000 Message-ID: <20060324002803.38643.qmail@minotaur.apache.org> Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r388306 - in /lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred: JobClient.java MapFileOutputFormat.java OutputFormat.java OutputFormatBase.java SequenceFileOutputFormat.java TextOutputFormat.java Date: Fri, 24 Mar 2006 00:28:01 -0000 To: hadoop-commits@lucene.apache.org From: cutting@apache.org X-Mailer: svnmailer-1.0.7 X-Virus-Checked: Checked by ClamAV on apache.org X-Spam-Rating: minotaur.apache.org 1.6.2 0/1000/N Author: cutting Date: Thu Mar 23 16:28:01 2006 New Revision: 388306 URL: http://svn.apache.org/viewcvs?rev=388306&view=rev Log: Move checking of output directory existence from JobClient to OutputFormat, so that it can be overridden. Add a base class for OutputFormat that implements this new method. Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormatBase.java Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormat.java lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TextOutputFormat.java Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java?rev=388306&r1=388305&r2=388306&view=diff ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java Thu Mar 23 16:28:01 2006 @@ -256,15 +256,8 @@ job.setWorkingDirectory(fileSys.getWorkingDirectory().toString()); } - // Ensure that the output directory is set and not already there - File outDir = job.getOutputDir(); - if (outDir == null && job.getNumReduceTasks() != 0) { - throw new IOException("Output directory not set in JobConf."); - } - if (outDir != null && fs.exists(outDir)) { - throw new IOException("Output directory " + outDir + - " already exists."); - } + // Check the output specification + job.getOutputFormat().checkOutputSpecs(fs, job); // Write job file to JobTracker's fs FSDataOutputStream out = fileSys.create(submitJobFile); Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java?rev=388306&r1=388305&r2=388306&view=diff ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java Thu Mar 23 16:28:01 2006 @@ -28,7 +28,7 @@ import org.apache.hadoop.conf.Configuration; /** An {@link OutputFormat} that writes {@link MapFile}s. */ -public class MapFileOutputFormat implements OutputFormat { +public class MapFileOutputFormat extends OutputFormatBase { public RecordWriter getRecordWriter(FileSystem fs, JobConf job, String name) throws IOException { Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormat.java URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormat.java?rev=388306&r1=388305&r2=388306&view=diff ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormat.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormat.java Thu Mar 23 16:28:01 2006 @@ -33,5 +33,15 @@ */ RecordWriter getRecordWriter(FileSystem fs, JobConf job, String name) throws IOException; + + /** Check whether the output specification for a job is appropriate. Called + * when a job is submitted. Typically checks that it does not already exist, + * throwing an exception when it already exists, so that output is not + * overwritten. + * + * @param job the job whose output will be written + * @throws IOException when output should not be attempted + */ + void checkOutputSpecs(FileSystem fs, JobConf job) throws IOException; } Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormatBase.java URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormatBase.java?rev=388306&view=auto ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormatBase.java (added) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormatBase.java Thu Mar 23 16:28:01 2006 @@ -0,0 +1,43 @@ +/** + * Copyright 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapred; + +import java.io.IOException; +import java.io.File; + +import org.apache.hadoop.fs.FileSystem; + +/** A base class for {@link OutputFormat}. */ +public abstract class OutputFormatBase implements OutputFormat { + public abstract RecordWriter getRecordWriter(FileSystem fs, + JobConf job, String name) + throws IOException; + + public void checkOutputSpecs(FileSystem fs, JobConf job) throws IOException { + // Ensure that the output directory is set and not already there + File outDir = job.getOutputDir(); + if (outDir == null && job.getNumReduceTasks() != 0) { + throw new IOException("Output directory not set in JobConf."); + } + if (outDir != null && fs.exists(outDir)) { + throw new IOException("Output directory " + outDir + + " already exists."); + } + } + +} + Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java?rev=388306&r1=388305&r2=388306&view=diff ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java Thu Mar 23 16:28:01 2006 @@ -28,7 +28,7 @@ import org.apache.hadoop.conf.Configuration; /** An {@link OutputFormat} that writes {@link SequenceFile}s. */ -public class SequenceFileOutputFormat implements OutputFormat { +public class SequenceFileOutputFormat extends OutputFormatBase { public RecordWriter getRecordWriter(FileSystem fs, JobConf job, String name) throws IOException { Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TextOutputFormat.java URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TextOutputFormat.java?rev=388306&r1=388305&r2=388306&view=diff ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TextOutputFormat.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TextOutputFormat.java Thu Mar 23 16:28:01 2006 @@ -26,7 +26,7 @@ import org.apache.hadoop.io.Writable; /** An {@link OutputFormat} that writes plain text files. */ -public class TextOutputFormat implements OutputFormat { +public class TextOutputFormat extends OutputFormatBase { public RecordWriter getRecordWriter(FileSystem fs, JobConf job, String name) throws IOException {