Return-Path: Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: (qmail 54012 invoked from network); 7 Dec 2010 16:32:42 -0000 Received: from unknown (HELO mail.apache.org) (140.211.11.3) by 140.211.11.9 with SMTP; 7 Dec 2010 16:32:42 -0000 Received: (qmail 52857 invoked by uid 500); 7 Dec 2010 16:32:42 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 52685 invoked by uid 500); 7 Dec 2010 16:32:40 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 52678 invoked by uid 99); 7 Dec 2010 16:32:39 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 07 Dec 2010 16:32:39 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 07 Dec 2010 16:32:38 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 4B00823888CE; Tue, 7 Dec 2010 16:32:18 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1043117 - in /hadoop/common/trunk: CHANGES.txt src/test/core/org/apache/hadoop/fs/loadGenerator/DataGenerator.java src/test/core/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java Date: Tue, 07 Dec 2010 16:32:18 -0000 To: common-commits@hadoop.apache.org From: sradia@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20101207163218.4B00823888CE@eris.apache.org> Author: sradia Date: Tue Dec 7 16:32:17 2010 New Revision: 1043117 URL: http://svn.apache.org/viewvc?rev=1043117&view=rev Log: HADOOP-7054 Change NN LoadGenerator to use FileContext APIs (Sanjay Radia) Modified: hadoop/common/trunk/CHANGES.txt hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/loadGenerator/DataGenerator.java hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java Modified: hadoop/common/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=1043117&r1=1043116&r2=1043117&view=diff ============================================================================== --- hadoop/common/trunk/CHANGES.txt (original) +++ hadoop/common/trunk/CHANGES.txt Tue Dec 7 16:32:17 2010 @@ -23,6 +23,9 @@ Trunk (unreleased changes) HADOOP-7049. TestReconfiguration should be junit v4. (Patrick Kling via eli) + HADOOP-7054 Change NN LoadGenerator to use FileContext APIs + (Sanjay Radia) + OPTIMIZATIONS BUG FIXES Modified: hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/loadGenerator/DataGenerator.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/loadGenerator/DataGenerator.java?rev=1043117&r1=1043116&r2=1043117&view=diff ============================================================================== --- hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/loadGenerator/DataGenerator.java (original) +++ hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/loadGenerator/DataGenerator.java Tue Dec 7 16:32:17 2010 @@ -22,12 +22,15 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; +import java.util.EnumSet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.Options.CreateOpts; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; @@ -48,7 +51,7 @@ import org.apache.hadoop.util.ToolRunner public class DataGenerator extends Configured implements Tool { private File inDir = StructureGenerator.DEFAULT_STRUCTURE_DIRECTORY; private Path root = DEFAULT_ROOT; - private FileSystem fs; + private FileContext fc; final static private long BLOCK_SIZE = 10; final static private String USAGE = "java DataGenerator " + "-inDir " + @@ -78,7 +81,7 @@ public class DataGenerator extends Confi /** Parse the command line arguments and initialize the data */ private int init(String[] args) { try { // initialize file system handle - fs = FileSystem.get(getConf()); + fc = FileContext.getFileContext(getConf()); } catch (IOException ioe) { System.err.println("Can not initialize the file system: " + ioe.getLocalizedMessage()); @@ -109,7 +112,7 @@ public class DataGenerator extends Confi StructureGenerator.DIR_STRUCTURE_FILE_NAME))); String line; while ((line=in.readLine()) != null) { - fs.mkdirs(new Path(root+line)); + fc.mkdir(new Path(root+line), FileContext.DEFAULT_PERM, true); } } @@ -137,10 +140,9 @@ public class DataGenerator extends Confi * a length of fileSize. The file is filled with character 'a'. */ private void genFile(Path file, long fileSize) throws IOException { - FSDataOutputStream out = fs.create(file, true, - getConf().getInt("io.file.buffer.size", 4096), - (short)getConf().getInt("dfs.replication", 3), - fs.getDefaultBlockSize()); + FSDataOutputStream out = fc.create(file, EnumSet.of(CreateFlag.OVERWRITE), + CreateOpts.createParent(), CreateOpts.bufferSize(4096), + CreateOpts.repFac((short) 3)); for(long i=0; i