hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject svn commit: r694458 [1/2] - in /hadoop/core/trunk: ./ docs/ src/docs/src/documentation/content/xdocs/ src/test/org/apache/hadoop/fs/loadGenerator/
Date Thu, 11 Sep 2008 20:23:18 GMT
Author: szetszwo
Date: Thu Sep 11 13:23:09 2008
New Revision: 694458

URL: http://svn.apache.org/viewvc?rev=694458&view=rev
Log:
HADOOP-3992 Add a synthetic load generation facility to the test directory. Contributed by Hairong Kuang.

Added:
    hadoop/core/trunk/docs/SLG_user_guide.html
    hadoop/core/trunk/docs/SLG_user_guide.pdf
    hadoop/core/trunk/src/docs/src/documentation/content/xdocs/SLG_user_guide.xml
    hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/
    hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/DataGenerator.java
    hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java
    hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/StructureGenerator.java
    hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/TestLoadGenerator.java
Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/docs/changes.html
    hadoop/core/trunk/docs/cluster_setup.html
    hadoop/core/trunk/docs/commands_manual.html
    hadoop/core/trunk/docs/distcp.html
    hadoop/core/trunk/docs/hadoop-default.html
    hadoop/core/trunk/docs/hadoop_archives.html
    hadoop/core/trunk/docs/hdfs_design.html
    hadoop/core/trunk/docs/hdfs_permissions_guide.html
    hadoop/core/trunk/docs/hdfs_quota_admin_guide.html
    hadoop/core/trunk/docs/hdfs_shell.html
    hadoop/core/trunk/docs/hdfs_user_guide.html
    hadoop/core/trunk/docs/hod.html
    hadoop/core/trunk/docs/hod_admin_guide.html
    hadoop/core/trunk/docs/hod_config_guide.html
    hadoop/core/trunk/docs/hod_user_guide.html
    hadoop/core/trunk/docs/index.html
    hadoop/core/trunk/docs/linkmap.html
    hadoop/core/trunk/docs/linkmap.pdf
    hadoop/core/trunk/docs/mapred_tutorial.html
    hadoop/core/trunk/docs/native_libraries.html
    hadoop/core/trunk/docs/quickstart.html
    hadoop/core/trunk/docs/streaming.html
    hadoop/core/trunk/src/docs/src/documentation/content/xdocs/site.xml

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Sep 11 13:23:09 2008
@@ -148,6 +148,9 @@
     HADOOP-3445. Add capacity scheduler that provides guaranteed capacities to 
     queues as a percentage of the cluster. (Vivek Ratan via omalley)
 
+    HADOOP-3992. Add a synthetic load generation facility to the test
+    directory. (hairong via szetszwo)
+
   IMPROVEMENTS
 
     HADOOP-3908. Fuse-dfs: better error message if llibhdfs.so doesn't exist.

Added: hadoop/core/trunk/docs/SLG_user_guide.html
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/SLG_user_guide.html?rev=694458&view=auto
==============================================================================
--- hadoop/core/trunk/docs/SLG_user_guide.html (added)
+++ hadoop/core/trunk/docs/SLG_user_guide.html Thu Sep 11 13:23:09 2008
@@ -0,0 +1,494 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<meta content="Apache Forrest" name="Generator">
+<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-skin-name" content="pelt">
+<title> Synthetic Load Generator User Guide </title>
+<link type="text/css" href="skin/basic.css" rel="stylesheet">
+<link media="screen" type="text/css" href="skin/screen.css" rel="stylesheet">
+<link media="print" type="text/css" href="skin/print.css" rel="stylesheet">
+<link type="text/css" href="skin/profile.css" rel="stylesheet">
+<script src="skin/getBlank.js" language="javascript" type="text/javascript"></script><script src="skin/getMenu.js" language="javascript" type="text/javascript"></script><script src="skin/fontsize.js" language="javascript" type="text/javascript"></script>
+<link rel="shortcut icon" href="images/favicon.ico">
+</head>
+<body onload="init()">
+<script type="text/javascript">ndeSetTextSize();</script>
+<div id="top">
+<!--+
+    |breadtrail
+    +-->
+<div class="breadtrail">
+<a href="http://www.apache.org/">Apache</a> &gt; <a href="http://hadoop.apache.org/">Hadoop</a> &gt; <a href="http://hadoop.apache.org/core/">Core</a><script src="skin/breadcrumbs.js" language="JavaScript" type="text/javascript"></script>
+</div>
+<!--+
+    |header
+    +-->
+<div class="header">
+<!--+
+    |start group logo
+    +-->
+<div class="grouplogo">
+<a href="http://hadoop.apache.org/"><img class="logoImage" alt="Hadoop" src="images/hadoop-logo.jpg" title="Apache Hadoop"></a>
+</div>
+<!--+
+    |end group logo
+    +-->
+<!--+
+    |start Project Logo
+    +-->
+<div class="projectlogo">
+<a href="http://hadoop.apache.org/core/"><img class="logoImage" alt="Hadoop" src="images/core-logo.gif" title="Scalable Computing Platform"></a>
+</div>
+<!--+
+    |end Project Logo
+    +-->
+<!--+
+    |start Search
+    +-->
+<div class="searchbox">
+<form action="http://www.google.com/search" method="get" class="roundtopsmall">
+<input value="hadoop.apache.org" name="sitesearch" type="hidden"><input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">&nbsp; 
+                    <input name="Search" value="Search" type="submit">
+</form>
+</div>
+<!--+
+    |end search
+    +-->
+<!--+
+    |start Tabs
+    +-->
+<ul id="tabs">
+<li>
+<a class="unselected" href="http://hadoop.apache.org/core/">Project</a>
+</li>
+<li>
+<a class="unselected" href="http://wiki.apache.org/hadoop">Wiki</a>
+</li>
+<li class="current">
+<a class="selected" href="index.html">Hadoop 0.19 Documentation</a>
+</li>
+</ul>
+<!--+
+    |end Tabs
+    +-->
+</div>
+</div>
+<div id="main">
+<div id="publishedStrip">
+<!--+
+    |start Subtabs
+    +-->
+<div id="level2tabs"></div>
+<!--+
+    |end Endtabs
+    +-->
+<script type="text/javascript"><!--
+document.write("Last Published: " + document.lastModified);
+//  --></script>
+</div>
+<!--+
+    |breadtrail
+    +-->
+<div class="breadtrail">
+
+             &nbsp;
+           </div>
+<!--+
+    |start Menu, mainarea
+    +-->
+<!--+
+    |start Menu
+    +-->
+<div id="menu">
+<div onclick="SwitchMenu('menu_selected_1.1', 'skin/')" id="menu_selected_1.1Title" class="menutitle" style="background-image: url('skin/images/chapter_open.gif');">Documentation</div>
+<div id="menu_selected_1.1" class="selectedmenuitemgroup" style="display: block;">
+<div class="menuitem">
+<a href="index.html">Overview</a>
+</div>
+<div class="menuitem">
+<a href="quickstart.html">Quickstart</a>
+</div>
+<div class="menuitem">
+<a href="cluster_setup.html">Cluster Setup</a>
+</div>
+<div class="menuitem">
+<a href="hdfs_design.html">HDFS Architecture</a>
+</div>
+<div class="menuitem">
+<a href="hdfs_user_guide.html">HDFS User Guide</a>
+</div>
+<div class="menuitem">
+<a href="hdfs_permissions_guide.html">HDFS Permissions Guide</a>
+</div>
+<div class="menuitem">
+<a href="hdfs_quota_admin_guide.html">HDFS Quotas Administrator Guide</a>
+</div>
+<div class="menuitem">
+<a href="commands_manual.html">Commands Manual</a>
+</div>
+<div class="menuitem">
+<a href="hdfs_shell.html">FS Shell Guide</a>
+</div>
+<div class="menupage">
+<div class="menupagetitle">Synthetic Load Generator User Guide</div>
+</div>
+<div class="menuitem">
+<a href="distcp.html">DistCp Guide</a>
+</div>
+<div class="menuitem">
+<a href="mapred_tutorial.html">Map-Reduce Tutorial</a>
+</div>
+<div class="menuitem">
+<a href="native_libraries.html">Native Hadoop Libraries</a>
+</div>
+<div class="menuitem">
+<a href="streaming.html">Streaming</a>
+</div>
+<div class="menuitem">
+<a href="hadoop_archives.html">Hadoop Archives</a>
+</div>
+<div class="menuitem">
+<a href="hod.html">Hadoop On Demand</a>
+</div>
+<div class="menuitem">
+<a href="api/index.html">API Docs</a>
+</div>
+<div class="menuitem">
+<a href="http://wiki.apache.org/hadoop/">Wiki</a>
+</div>
+<div class="menuitem">
+<a href="http://wiki.apache.org/hadoop/FAQ">FAQ</a>
+</div>
+<div class="menuitem">
+<a href="http://hadoop.apache.org/core/mailing_lists.html">Mailing Lists</a>
+</div>
+<div class="menuitem">
+<a href="releasenotes.html">Release Notes</a>
+</div>
+<div class="menuitem">
+<a href="changes.html">All Changes</a>
+</div>
+</div>
+<div id="credit"></div>
+<div id="roundbottom">
+<img style="display: none" class="corner" height="15" width="15" alt="" src="skin/images/rc-b-l-15-1body-2menu-3menu.png"></div>
+<!--+
+  |alternative credits
+  +-->
+<div id="credit2"></div>
+</div>
+<!--+
+    |end Menu
+    +-->
+<!--+
+    |start content
+    +-->
+<div id="content">
+<div title="Portable Document Format" class="pdflink">
+<a class="dida" href="SLG_user_guide.pdf"><img alt="PDF -icon" src="skin/images/pdfdoc.gif" class="skin"><br>
+        PDF</a>
+</div>
+<h1> Synthetic Load Generator User Guide </h1>
+<div id="minitoc-area">
+<ul class="minitoc">
+<li>
+<a href="#Description"> Description </a>
+</li>
+<li>
+<a href="#Synopsis"> Synopsis </a>
+</li>
+<li>
+<a href="#Test+Space+Population"> Test Space Population </a>
+<ul class="minitoc">
+<li>
+<a href="#Structure+Generator"> Structure Generator </a>
+</li>
+<li>
+<a href="#Test+Space+Generator"> Test Space Generator </a>
+</li>
+</ul>
+</li>
+</ul>
+</div>
+		
+<a name="N1000D"></a><a name="Description"></a>
+<h2 class="h3"> Description </h2>
+<div class="section">
+<p>
+        The synthetic load generator is a tool for testing NameNode behavior
+        under different client loads. The user can generate different mixes 
+        of read, write, and list requests by specifying the probabilities of
+        read and write. The user controls the intensity of the load by adjusting
+        parameters for the number of worker threads and the delay between 
+        operations. While load generators are running, the user can profile and
+        monitor the running of the NameNode. When a load generator exits, it
+        prints some NameNode statistics like the average execution time of each
+        kind of operation and the NameNode throughput.
+                       </p>
+</div>
+		
+<a name="N10017"></a><a name="Synopsis"></a>
+<h2 class="h3"> Synopsis </h2>
+<div class="section">
+<p>
+        
+<span class="codefrag">java LoadGenerator [options]</span>
+<br>
+                        
+</p>
+<p>
+        Options include:<br>
+        
+<span class="codefrag">&nbsp;&nbsp;-readProbability &lt;read probability&gt;</span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;the probability of the read operation;
+                default is 0.3333. </span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;-writeProbability &lt;write probability&gt;</span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;the probability of the write 
+                operations; default is 0.3333.</span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;-root &lt;test space root&gt;</span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;the root of the test space;
+                default is /testLoadSpace.</span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;-maxDelayBetweenOps 
+                &lt;maxDelayBetweenOpsInMillis&gt;</span>
+<br> 
+        
+<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;the maximum delay between two consecutive
+                operations in a thread; default is 0 indicating no delay.
+                </span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;-numOfThreads &lt;numOfThreads&gt;</span>
+<br> 
+        
+<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;the number of threads to spawn; 
+                default is 200.</span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;-elapsedTime &lt;elapsedTimeInSecs&gt;</span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;the number of seconds that the program 
+                will run; A value of zero indicates that the program runs
+                forever. The default value is 0.</span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;-startTime &lt;startTimeInMillis&gt;</span>
+<br> 
+        
+<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;the time that all worker threads 
+                start to run. By default it is 10 seconds after the main 
+                program starts running.This creates a barrier if more than
+                one load generator is running.
+        </span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;-seed &lt;seed&gt;</span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;the random generator seed for repeating 
+                requests to NameNode when running with a single thread;
+                default is the current time.</span>
+<br>
+			
+</p>
+<p>
+        After command line argument parsing, the load generator traverses 
+        the test space and builds a table of all directories and another table
+        of all files in the test space. It then waits until the start time to
+        spawn the number of worker threads as specified by the user. Each
+        thread sends a stream of requests to NameNode. At each iteration, 
+        it first decides if it is going to read a file, create a file, or
+        list a directory following the read and write probabilities specified
+        by the user. The listing probability is equal to 
+        <em>1-read probability-write probability</em>. When reading, 
+        it randomly picks a file in the test space and reads the entire file. 
+        When writing, it randomly picks a directory in the test space and 
+        creates a file there. To avoid two threads with the same load 
+        generator or from two different load generators create the same 
+        file, the file name consists of the current machine's host name 
+        and the thread id. The length of the file follows Gaussian 
+        distribution with an average size of 2 blocks and the standard 
+        deviation of 1. The new file is filled with byte 'a'. To avoid
+        the test space to grow indefinitely, the file is deleted immediately
+        after the file creation completes. While listing, it randomly 
+        picks a directory in the test space and lists its content. 
+        After an operation completes, the thread pauses for a random 
+        amount of time in the range of [0, maxDelayBetweenOps] if the 
+        specified maximum delay is not zero. All threads are stopped when 
+        the specified elapsed time is passed. Before exiting, the program 
+        prints the average execution for each kind of NameNode operations, 
+        and the number of requests served by the NameNode per second.
+                        </p>
+</div>
+                
+<a name="N10070"></a><a name="Test+Space+Population"></a>
+<h2 class="h3"> Test Space Population </h2>
+<div class="section">
+<p>
+        The user needs to populate a test space before she runs a 
+        load generator. The structure generator generates a random 
+        test space structure and the data generator creates the files 
+        and directories of the test space in Hadoop distributed file system.
+                        </p>
+<a name="N10079"></a><a name="Structure+Generator"></a>
+<h3 class="h4"> Structure Generator </h3>
+<p>
+        This tool generates a random namespace structure with the 
+        following constraints:
+                                </p>
+<ol>
+        
+<li>The number of subdirectories that a directory can have is 
+            a random number in [minWidth, maxWidth].</li>
+        
+<li>The maximum depth of each subdirectory is a random number 
+            [2*maxDepth/3, maxDepth].</li>
+        
+<li>Files are randomly placed in leaf directories. The size of 
+            each file follows Gaussian distribution with an average size 
+            of 1 block and a standard deviation of 1.</li>
+                                        
+</ol>
+<p>
+        The generated namespace structure is described by two files in 
+        the output directory. Each line of the first file contains the 
+        full name of a leaf directory. Each line of the second file 
+        contains the full name of a file and its size, separated by a blank.
+                                </p>
+<p>
+        The synopsis of the command is
+                                </p>
+<p>
+        
+<span class="codefrag">java StructureGenerator [options]</span>
+                                
+</p>
+<p>
+        Options include:<br>
+        
+<span class="codefrag">&nbsp;&nbsp;-maxDepth &lt;maxDepth&gt;</span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;maximum depth of the directory tree; 
+                default is 5.</span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;-minWidth &lt;minWidth&gt;</span>
+<br> 
+        
+<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;minimum number of subdirectories per 
+                directories; default is 1.</span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;-maxWidth &lt;maxWidth&gt;</span>
+<br> 
+        
+<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;maximum number of subdirectories per 
+                directories; default is 5.</span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;-numOfFiles &lt;#OfFiles&gt;</span>
+<br> 
+        
+<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;the total number of files in the test 
+                space; default is 10.</span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;-avgFileSize &lt;avgFileSizeInBlocks&gt;</span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;average size of blocks; default is 1.
+                </span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;-outDir &lt;outDir&gt;</span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;output directory; default is the 
+                current directory. </span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;-seed &lt;seed&gt;</span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;random number generator seed; 
+                default is the current time.</span>
+<br>
+                                
+</p>
+<a name="N100D8"></a><a name="Test+Space+Generator"></a>
+<h3 class="h4"> Test Space Generator </h3>
+<p>
+        This tool reads the directory structure and file structure from 
+        the input directory and creates the namespace in Hadoop distributed
+        file system. All files are filled with byte 'a'.
+                                </p>
+<p>
+        The synopsis of the command is
+                                </p>
+<p>
+        
+<span class="codefrag">java DataGenerator [options]</span>
+                                
+</p>
+<p>
+        Options include:<br>
+        
+<span class="codefrag">&nbsp;&nbsp;-inDir &lt;inDir&gt;</span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;input directory name where directory/file
+                structures are stored; default is the current directory.
+        </span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;-root &lt;test space root&gt;</span>
+<br>
+        
+<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;the name of the root directory which the 
+                new namespace is going to be placed under; 
+                default is "/testLoadSpace".</span>
+<br>
+                                
+</p>
+</div>
+	
+</div>
+<!--+
+    |end content
+    +-->
+<div class="clearboth">&nbsp;</div>
+</div>
+<div id="footer">
+<!--+
+    |start bottomstrip
+    +-->
+<div class="lastmodified">
+<script type="text/javascript"><!--
+document.write("Last Published: " + document.lastModified);
+//  --></script>
+</div>
+<div class="copyright">
+        Copyright &copy;
+         2008 <a href="http://www.apache.org/licenses/">The Apache Software Foundation.</a>
+</div>
+<!--+
+    |end bottomstrip
+    +-->
+</div>
+</body>
+</html>

Added: hadoop/core/trunk/docs/SLG_user_guide.pdf
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/SLG_user_guide.pdf?rev=694458&view=auto
==============================================================================
--- hadoop/core/trunk/docs/SLG_user_guide.pdf (added)
+++ hadoop/core/trunk/docs/SLG_user_guide.pdf Thu Sep 11 13:23:09 2008
@@ -0,0 +1,314 @@
+%PDF-1.3
+%ª«¬­
+4 0 obj
+<< /Type /Info
+/Producer (FOP 0.20.5) >>
+endobj
+5 0 obj
+<< /Length 546 /Filter [ /ASCII85Decode /FlateDecode ]
+ >>
+stream
+Gaua<9iHZu&;KZOMB(B'CH`$NUK-%da+FkH;@c=IQC1>VJkl?Z#uD6sZoqPO^kU+JH$5B_Sp8ipL8YV"-#4$\8*46enHd#.->7#2!ZqLP,*[JLf1EV.e5iM1S"_DO7f)`;;5PjZ/O^h_<6<8m^!sp(Q;,(+;tQ*5#;%#Ad`[\.H,tQ@T[QT_O.@s*\G7tDC\s^F6A\%Z6TgA7*lfLmRXgoY+*R11lV7D(\'nXO4&KWj?E.6G<YmB1knOarJ=.oQ'7-+:0q^Z94jEA=lUo@+cKiY@+2A!8JY0nU38gB]S`Y+.'2^:'*1(C%m(FcL2S8T*HU/$/CoBHXRK7KVr)j$I+Tl\*cVk$qj<(G&a])2!jJ-epTAW2"7>K$X_6)Hc0Q8,io,Y=#J$encb(*OarLrY.cZ>_'?Yui]gEjZZKGp[m'aDfX-.(@mp?&6kdihX,+r^=5l0s@IQj"JSF=R,4graYEjdWBX4o:4Z@P*IWA?BAmXDt"6k1[(FX.Qb>7M`9Y<p^1pBA.69NE3Z7Cn^kNhdQa)r>1lrBF=~>
+endstream
+endobj
+6 0 obj
+<< /Type /Page
+/Parent 1 0 R
+/MediaBox [ 0 0 612 792 ]
+/Resources 3 0 R
+/Contents 5 0 R
+/Annots 7 0 R
+>>
+endobj
+7 0 obj
+[
+8 0 R
+10 0 R
+12 0 R
+14 0 R
+16 0 R
+]
+endobj
+8 0 obj
+<< /Type /Annot
+/Subtype /Link
+/Rect [ 102.0 559.666 165.992 547.666 ]
+/C [ 0 0 0 ]
+/Border [ 0 0 0 ]
+/A 9 0 R
+/H /I
+>>
+endobj
+10 0 obj
+<< /Type /Annot
+/Subtype /Link
+/Rect [ 102.0 541.466 153.344 529.466 ]
+/C [ 0 0 0 ]
+/Border [ 0 0 0 ]
+/A 11 0 R
+/H /I
+>>
+endobj
+12 0 obj
+<< /Type /Annot
+/Subtype /Link
+/Rect [ 102.0 523.266 217.328 511.266 ]
+/C [ 0 0 0 ]
+/Border [ 0 0 0 ]
+/A 13 0 R
+/H /I
+>>
+endobj
+14 0 obj
+<< /Type /Annot
+/Subtype /Link
+/Rect [ 108.0 505.066 218.468 493.066 ]
+/C [ 0 0 0 ]
+/Border [ 0 0 0 ]
+/A 15 0 R
+/H /I
+>>
+endobj
+16 0 obj
+<< /Type /Annot
+/Subtype /Link
+/Rect [ 108.0 486.866 226.796 474.866 ]
+/C [ 0 0 0 ]
+/Border [ 0 0 0 ]
+/A 17 0 R
+/H /I
+>>
+endobj
+18 0 obj
+<< /Length 1807 /Filter [ /ASCII85Decode /FlateDecode ]
+ >>
+stream
+GatU49on$e&A@sBka2?Q7b"p%TKor"JeVp^@U8Vfk!2S(bsW<fM:u80f68Lfim=Q-Vo9]X6]:$<2ifa^h6%O>?`+s)M!qIgL$rjoeFVX[H:o@"OshS49YAp$qt,Q)Vqt,ZpZ#%UnMY=(?3sb!ofd`'?PgJY(>iM?^VH>X3&upgQ6]cn9Y$k2:Em"$hKn_hijR`(p1Q-tCi:4g&uFW[X"?9"[YJ%'\IuFjW['*R[G`u]R$*FH'!6Fkk#"]Q)6:\3'Q9882eLu2g[Y$D5`VrAQJDp2WN-DebX.O?Dg8D.:gT!p*ltYWqABSrhB#INGdd*c/`%^bobhe!d=n-9jG4:KRcfF3igDMC)PK-r=cHOd7'#h$D]LNdem0@oaS%_4jh]qTopF$=LmKjd2%BFPY1ZuX6A\>VcT!n/IjKrW<fO37'CR:@e;Dk[:^=H?f"8@)0aXrVdV2E9?tY^78(amDXXW*LIr&u]%I7o@``THX&$l#RXbf2Jm0mFSZ3@EKEdl9`"6W3iEo/AV%R/J.T3G0`ET$neA0fQ1Vo+D*C="/!=4]^kZPbt\S-G'NC&)84h8rj!=]Y3u<s8Gg(G$JLmDL$c;bCP@P&Y"cr[:a(#S7>q"EbpV>.J]6LON;923;7,k#C=jT0^Nc_3kbQ^@8cS'E0gSi%9;M*71Hgb&3a2>P%hqJYRg2Sg8K=?I+acfO_PFmQ3kE8%?XnhGHlX58`2a:aGK>[G86>opD^rYof,_%$&ji9L9@oeSXQRDGdhfb@=-B_X;b2,lY%24-7?!,Q^.,8IfmOrB,Y!.@_!?Y4T6rLN*sPAU2f@r.f!OEsJZd6&CUeXRU:uRm(b<SC6rD\Fi1$?u;?sIJE%p#F&gGo-+`4G[,ENi;$>.39WXm>Ar9N:/^o8N"s!9?)>c\ciiQF+.NU(EpDY?4BS%qk@Q+eDWnu6hh.K>-KtI3qoQQ!(&o6Lo5bfFJJ$'O;F1pGP(T!4knecSAB]5CEZ?N
 Qf?HgQ-8Si0f:OA+Z1#(%Gi$d6O4ml(CBej3qVI\^H[O?[`S7X3Ib:FG?C_&j[f0!#cm6k$O-VS"J<e'UgL1q/-.\rm`((F]UbSO+Bnlh>mh?&e!j07Cb(.8EAW2VdeV=NN;i$t.:0N_/nc[_Vqa$#P#AqbAE]*MC8)Ee*%+E^u&_YYo3G!B>PEM6Y*j3dF!W7#u$^N5>(\"YIH*J9'kn+sTI9YV!Hj4)BKemJTKFrk]CCB0g.;[ZDC'.`jb!@.fFC%&Qa%Q#Kc`5:Dk6Iu)jFt;DRlBn35Q!`l$%AS9cpVs=B8@F.iRq-H*T"f8dA"YA/h??d8!23W%CBN&3XPF8FuaCB+Z5^n;k^Ujbn7jeRdL<"8fq;X$QnS#jP:l[j,U!+S7:WF;#c,"=fH4uX`TCUfpMgE0Wg&%n*A)1l`Q!D`+tNI%a8&'6XAE723#CX$)HOND!YOP(M/STd]L9m2WnA<E;8jc(gU%(/R-L.X=c3_Jtcs;=+[-tQFi(#ft3LDVhX[98_Q\,#`uF=6U6jfBH4GQ8,[=FnMBpH.5.[&1LMfI<`r;qSru#'[=ZEJBKPZ$rM9[g8TXV3]F<eQqWtF>!)`*Z'\\LR3A,?-^M]leEmc3Mmf_2A^*&fj5P/O>][hpkj3(*qD/](%:IuJjH]([O\A,G9a'/p7oecpq_^tGG&]mr<l:67WS9>8O<nab3InV+jO9.uE8Wgm/@[Y/@('//20(lR+*u_E;%Mr?2,`qb%&6!()(!V]65NRjD95*7fI4%\\TKaTY,HkZ[(IJ9md'OrFC1Go.75bAtms.2-GQ.YZ2CYX~>
+endstream
+endobj
+19 0 obj
+<< /Type /Page
+/Parent 1 0 R
+/MediaBox [ 0 0 612 792 ]
+/Resources 3 0 R
+/Contents 18 0 R
+>>
+endobj
+20 0 obj
+<< /Length 2367 /Filter [ /ASCII85Decode /FlateDecode ]
+ >>
+stream
+Gau0F968iG&AII3kSC([0RQga/T,(-SPGW6e]GYP!>6O^A!14kd#.D[]+eVa'HjYu4!$UW&c;@-qs?$"gXYZ.CA-9'RIB)2I^ZU8HQInbr:GX<n$fGA=Q-pNoC_JEg$NPJmOjkg/pstYcKs"0)nqhfZSHL%d.ZY'FT-OVI]I_`)C1gUh/$VjZ7I&7,U29d'e=H-atkp1a]VJuD"hPOU\8@e+rM$%iu'M;P";8rfJ>0Q?5`qp?6X5tc8O;pD!AMD8W:TlfCLZeeLr0)]qsh7^+6WeO>OL:Mj%\]Ldf-=@(!dN'&>+&?o<+CU?J^MYY^HVLVXn3Io^(t&oqAH+o,-86o%Z6!Qbd&[f`U/XMEgaMTq=&W.W4iX@h(]*fS-*(/G(EIDHfSa:^#^)mL+tn,(M*Zmp?P5l/8Sqlpn^QtNg%%2b+<fc:n</PmQ4e[nfM^FIm?;@h8IQL^(l`Z+[o+j>e66IcquAG-5b_Z<`paG9kgQpfG)^J:G7FqQQ4D^tu_Ro7(CBe;Arl?GdaDIC0;6i`6poH`RP6suA.RP#3"Nc/Kla9/gl#JnI^S@"&<@,(6=Y1Zu(6BOC&&oegd(?*'6UHi&rJcoG;on+,PrGFo;Z%VPr8cQE7Yikr0(,oE!@H^X2S@91HV"(Q!P@LdB37HkS6)de9BL*,X-BqI[V\Pte9+?Q:2c+Vt4"&J[6'@/N;e_2lQVjb+:e#h#Ud-YfTVT_iY1Mm/]ABUQIFNcZg&iXj?.U@CJ)Gg?`DaRTM]d1mKrBYf97_brK>d(F(a"Fb)sf1[W_YW>)Mn6)hgtga(,@`a(5qPq>f%MB6$,-,l_=1Z(*l[RBJYk,Si>$b7V8J>Ka3qj,(3B)e?9SO%Q&O9d,MaW+d*G\8VGI+)O4>8)p+.R41rm/@jr*$kA.,@i7\8:h6X+4"j?CY6J3.?W\16I@)L=B]d_^ph%SZSD4QUM6R$5r)PMcF.Z7G\5*#HF@+7<?`>M%BJ\[9
 MhTYc28;nkj,f(4U,-jDlBR[p,:!E+KCkFg0>p5?P,)c7tA5JMRj0UqbDT+`sWqP`u-%G.iKIYE7`rJ3A"btFHk^prU^6:.$0e0A,Q_Y=W7;_"u=pqLb3s4__HsSc76b]c?,a;WVF91a'J5U>n9/4Aa-1eRMeY$O.kqTO*OPmW$D)&),RoM\3#.A>57OqA>@7DMa7-5P#;$E(D"$o''aK:\TrK/JoGg"WgoB`s!r.:`6BMd09Jfp1VWRf\:+1O#-[HY"#B%Q#$)_WJn^a'f8O+XYZ>UK'4O9k*!XuQRWjX%Vq6G17k?G_oIb$,?;Y9cQnG6Th`?cVOM0aaB*?%GMW]F^Jd21B27JUcdN;.1X<8E[Q3J[5^(8N.]*;?\Gf.4[7c]8M6l-I:W5""T<]c]b6mF)uE(V<#E!oj5&T6Ss"7MbFX/Zb9p\"^]iKmG[n.R-B/WdjLPb8P:6Tq<,e/hI:30@FuZd_66;c)32k<8PrkIU;iqXpfmj9b'@e?o^g'brVV#)En>.npE('/WA8lF0Qa1hmL*3lC838,icq/4Krt*<p:7kJ(^e:$HI5f)L:6*'BT&9f)AboOfQ`iE3c6b7OQS8K^\(lZp1N_j004uK`_S.O1<28bD+nrpd4V1VQAVRcP<<.'3_LuZ(0r9GL@O4Bl2(KI;Z\t>'foZf5RpKLmZd?`QU!r6T0IqgLeM+n9(`a94b,/DO.$0GH>6=]X3(A<*aZnFf#"kIAa>Xl>rWR=m)A47kd_ZRNEkK'Z<D#G:[QirPHbSb[e<P#oYpG8eftRK]#rUC\u4guHq[!&=i>&NS"8$10@j-FTk/4)FPR8eeh)mth7bQ-&`U?8e>/?I@;fhHrW-PjmWYR0%s5SXhWWq!>EmEgmbnIK\WIVL:jE_IE`#$WNHc#tNb7oS+Q$WH#CXkUCjR4XJkSA8l51m^\daCV^F-15[b!h1"@LW9O#+P'.X!uH)_)\MC[cq,s1CZaNXR@,i9f
 /2Bp_AXFD`]i!1C+l_^AtnhdFPujcqk;Q)!ja:%RXIj88s)rIlI@nmZ,dZ,<@EHuEg7.QmFOF"d)6mogc;F7*d1dBq6Oh9_f/+^4]NhGL[+\B,%BSLSQ^of3rp,E="E@%FOg8K[6eJ?'.PnsIP3'N=k6Ze*p*lj;.30.d-^?6a-d*-^)7I5kg]:5M[s+H!73'Y<n*?e]b9^N#=hdJiPUmC4i4<f*4(I]IobhtUpgV[%(>"`=QfUUBJ-/OsU?dYB9J[I[>?>J2KsH0NsU=b$OoNVieT=+CqeehRZrOB7J.1noG\q6:LCr@*GpRT+5)F9?0cEc$.X>dYO">eC<I_:K3<Wt'!+C$scu$>?YU4UWiW_Qp^3kq:@@D^J@t!:Fb9&,V+~>
+endstream
+endobj
+21 0 obj
+<< /Type /Page
+/Parent 1 0 R
+/MediaBox [ 0 0 612 792 ]
+/Resources 3 0 R
+/Contents 20 0 R
+>>
+endobj
+22 0 obj
+<< /Length 1372 /Filter [ /ASCII85Decode /FlateDecode ]
+ >>
+stream
+Gau`S>E@f$'RoMSLmEgeIAeN:i`r-Cg0p?LKX@BfnZ_@IR^fICND&8).DMI<>CAV.Ah$mh<NU9]qqSo,cKF&dB&Bb'J'P"!L%"%QG)>Mf5leY`Dk?-@kS';0:+;_+m-!'XWs9II&__^2&3H,mS(2-5[\mC*:o?Cs)$sR*=m_c@2np_9aZF,V\BUM4V(V(5f;3^+Kd[V:9(VJG3bYLMlI.iXPI()dD"+b?>JE[b5Nr-o*<J+"FnJ3>BjPH3bHmN0647pai]@1##TM:;pW6e:ih1M8T>e&,Fg((X*"GG5i,[fea'l%(f6_QuZZW7,H.G'.p;p4.OG4Ep?OH<<f!J$6>akn"R:f9dqppMA@nIpYROioQR2oJG1`dPG,6K')[6lJ0rE>HGYuUICLe%m6k5cYi43$9e*,5N;!B5PhnEE@9:k0=e(ZG*]'B+<.$cr;XiA5Au]EWeAqJQlCIb6d(*;d);U0eg7J02fYhc@:&V3Chg>\n5?ZC'hU/Lj`TeKILkXo?KXq,(N2Vs\e9r6R70cBG/CPCfY/)`b5iX5=_;\Oumhaq9[D.*3,2ZRsU(=^#fKeud55qg5'QI6Tt&X'4CRKYjHfR./l%%(<D1;!Z<Q4C.9FKk+ARV's@59glg&e>AKB)2b+"-R*Ls"(R`5R<kPg\!K5+B`:-aBt5&@bVPq[]b.^D_N7U:3d3a(LnUhK=8KV;CeET@L`ot)QON6b0qj%UE&7_cbi7gmrq&/t+fCl$X.;Z>WKkVc</Ms=06bK#9O.No23>FQ!d'CJ9S=4+Dorj<0lsYa$?b(6AL[l8[29o!T$IZaU:4qQa^.q)#jp.,T>+#SV%2,_#XPV//Z*#?<Ee`$ipZKohM]gO=K.C-NqFH)945H?5&%CuD=[CGK6<;38j%a4j]A,c'aDAm;.GGYeUDG5p,mlGDH<<6kRKZ+[I8V>M,+&neW7Pk,T]qbql:eH:^>3U@?8`8*hPTSUZ*/+Er%9;<0WN
 *4&M#DrGMM,)<TDp(haDe[;h$WK:C"$i<j[TY0$C[jCTqs#gZ/?d*]f<Tc:8i4-<]s.^o4p1lIQ!C"nUZ1laYcmNrD[ib09Yk4eYYK>L(A>Xr!EM_2Xm+jrlL%QPq,9NN$u.'H),c#&[#+'"K3D"5NL%s&op#$&5N"7Q$?$%\?iM`W#IVZO`d12WHap`^1Mk1hfiX9ICRB(_K_fF*bJbQOf@k-&0Z(uXV>pWNakcPp3E%(o!#qoZ`F0CkXc-W/O.qGVU31=gge.s8J4G"KV=#h&VQV*Jp<^n&Q%W-`'h;28(!\%@qk@"LWfIq+ueM7IM7Ujc9d_L'?[QBu?]O8Z$F<ZVr?Qo(RoTOZfNmtB88Kk(4PHkI;O\bcVn7`hP~>
+endstream
+endobj
+23 0 obj
+<< /Type /Page
+/Parent 1 0 R
+/MediaBox [ 0 0 612 792 ]
+/Resources 3 0 R
+/Contents 22 0 R
+>>
+endobj
+25 0 obj
+<<
+ /Title (\376\377\0\61\0\40\0\104\0\145\0\163\0\143\0\162\0\151\0\160\0\164\0\151\0\157\0\156)
+ /Parent 24 0 R
+ /Next 26 0 R
+ /A 9 0 R
+>> endobj
+26 0 obj
+<<
+ /Title (\376\377\0\62\0\40\0\123\0\171\0\156\0\157\0\160\0\163\0\151\0\163)
+ /Parent 24 0 R
+ /Prev 25 0 R
+ /Next 27 0 R
+ /A 11 0 R
+>> endobj
+27 0 obj
+<<
+ /Title (\376\377\0\63\0\40\0\124\0\145\0\163\0\164\0\40\0\123\0\160\0\141\0\143\0\145\0\40\0\120\0\157\0\160\0\165\0\154\0\141\0\164\0\151\0\157\0\156)
+ /Parent 24 0 R
+ /First 28 0 R
+ /Last 29 0 R
+ /Prev 26 0 R
+ /Count -2
+ /A 13 0 R
+>> endobj
+28 0 obj
+<<
+ /Title (\376\377\0\63\0\56\0\61\0\40\0\123\0\164\0\162\0\165\0\143\0\164\0\165\0\162\0\145\0\40\0\107\0\145\0\156\0\145\0\162\0\141\0\164\0\157\0\162)
+ /Parent 27 0 R
+ /Next 29 0 R
+ /A 15 0 R
+>> endobj
+29 0 obj
+<<
+ /Title (\376\377\0\63\0\56\0\62\0\40\0\124\0\145\0\163\0\164\0\40\0\123\0\160\0\141\0\143\0\145\0\40\0\107\0\145\0\156\0\145\0\162\0\141\0\164\0\157\0\162)
+ /Parent 27 0 R
+ /Prev 28 0 R
+ /A 17 0 R
+>> endobj
+30 0 obj
+<< /Type /Font
+/Subtype /Type1
+/Name /F3
+/BaseFont /Helvetica-Bold
+/Encoding /WinAnsiEncoding >>
+endobj
+31 0 obj
+<< /Type /Font
+/Subtype /Type1
+/Name /F5
+/BaseFont /Times-Roman
+/Encoding /WinAnsiEncoding >>
+endobj
+32 0 obj
+<< /Type /Font
+/Subtype /Type1
+/Name /F6
+/BaseFont /Times-Italic
+/Encoding /WinAnsiEncoding >>
+endobj
+33 0 obj
+<< /Type /Font
+/Subtype /Type1
+/Name /F1
+/BaseFont /Helvetica
+/Encoding /WinAnsiEncoding >>
+endobj
+34 0 obj
+<< /Type /Font
+/Subtype /Type1
+/Name /F9
+/BaseFont /Courier
+/Encoding /WinAnsiEncoding >>
+endobj
+35 0 obj
+<< /Type /Font
+/Subtype /Type1
+/Name /F2
+/BaseFont /Helvetica-Oblique
+/Encoding /WinAnsiEncoding >>
+endobj
+36 0 obj
+<< /Type /Font
+/Subtype /Type1
+/Name /F7
+/BaseFont /Times-Bold
+/Encoding /WinAnsiEncoding >>
+endobj
+1 0 obj
+<< /Type /Pages
+/Count 4
+/Kids [6 0 R 19 0 R 21 0 R 23 0 R ] >>
+endobj
+2 0 obj
+<< /Type /Catalog
+/Pages 1 0 R
+ /Outlines 24 0 R
+ /PageMode /UseOutlines
+ >>
+endobj
+3 0 obj
+<< 
+/Font << /F3 30 0 R /F5 31 0 R /F1 33 0 R /F6 32 0 R /F9 34 0 R /F2 35 0 R /F7 36 0 R >> 
+/ProcSet [ /PDF /ImageC /Text ] >> 
+endobj
+9 0 obj
+<<
+/S /GoTo
+/D [19 0 R /XYZ 85.0 659.0 null]
+>>
+endobj
+11 0 obj
+<<
+/S /GoTo
+/D [19 0 R /XYZ 85.0 527.466 null]
+>>
+endobj
+13 0 obj
+<<
+/S /GoTo
+/D [21 0 R /XYZ 85.0 413.4 null]
+>>
+endobj
+15 0 obj
+<<
+/S /GoTo
+/D [21 0 R /XYZ 85.0 334.666 null]
+>>
+endobj
+17 0 obj
+<<
+/S /GoTo
+/D [23 0 R /XYZ 85.0 392.2 null]
+>>
+endobj
+24 0 obj
+<<
+ /First 25 0 R
+ /Last 27 0 R
+>> endobj
+xref
+0 37
+0000000000 65535 f 
+0000009488 00000 n 
+0000009567 00000 n 
+0000009659 00000 n 
+0000000015 00000 n 
+0000000071 00000 n 
+0000000708 00000 n 
+0000000828 00000 n 
+0000000881 00000 n 
+0000009804 00000 n 
+0000001016 00000 n 
+0000009867 00000 n 
+0000001153 00000 n 
+0000009933 00000 n 
+0000001290 00000 n 
+0000009997 00000 n 
+0000001427 00000 n 
+0000010063 00000 n 
+0000001564 00000 n 
+0000003464 00000 n 
+0000003572 00000 n 
+0000006032 00000 n 
+0000006140 00000 n 
+0000007605 00000 n 
+0000010127 00000 n 
+0000007713 00000 n 
+0000007870 00000 n 
+0000008024 00000 n 
+0000008280 00000 n 
+0000008495 00000 n 
+0000008715 00000 n 
+0000008828 00000 n 
+0000008938 00000 n 
+0000009049 00000 n 
+0000009157 00000 n 
+0000009263 00000 n 
+0000009379 00000 n 
+trailer
+<<
+/Size 37
+/Root 2 0 R
+/Info 4 0 R
+>>
+startxref
+10178
+%%EOF

Modified: hadoop/core/trunk/docs/changes.html
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/changes.html?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/docs/changes.html (original)
+++ hadoop/core/trunk/docs/changes.html Thu Sep 11 13:23:09 2008
@@ -94,7 +94,7 @@
     </ol>
   </li>
   <li><a href="javascript:toggleList('trunk_(unreleased_changes)_._new_features_')">  NEW FEATURES
-</a>&nbsp;&nbsp;&nbsp;(23)
+</a>&nbsp;&nbsp;&nbsp;(29)
     <ol id="trunk_(unreleased_changes)_._new_features_">
       <li><a href="http://issues.apache.org/jira/browse/HADOOP-3341">HADOOP-3341</a>. Allow streaming jobs to specify the field separator for map
 and reduce input and output. The new configuration values are:
@@ -147,10 +147,20 @@
       <li><a href="http://issues.apache.org/jira/browse/HADOOP-3866">HADOOP-3866</a>. Added sort and multi-job updates in the JobTracker web ui.<br />(Craig Weisenfluh via omalley)</li>
       <li><a href="http://issues.apache.org/jira/browse/HADOOP-3698">HADOOP-3698</a>. Add access control to control who is allowed to submit or
 modify jobs in the JobTracker.<br />(Hemanth Yamijala via omalley)</li>
+      <li><a href="http://issues.apache.org/jira/browse/HADOOP-1869">HADOOP-1869</a>. Support access times for HDFS files.<br />(dhruba)</li>
+      <li><a href="http://issues.apache.org/jira/browse/HADOOP-3941">HADOOP-3941</a>. Extend FileSystem API to return file-checksums.<br />(szetszwo)</li>
+      <li><a href="http://issues.apache.org/jira/browse/HADOOP-3581">HADOOP-3581</a>. Prevents memory intensive user tasks from taking down
+nodes.<br />(Vinod K V via ddas)</li>
+      <li><a href="http://issues.apache.org/jira/browse/HADOOP-3970">HADOOP-3970</a>. Provides a way to recover counters written to JobHistory.<br />(Amar Kamat via ddas)</li>
+      <li><a href="http://issues.apache.org/jira/browse/HADOOP-3702">HADOOP-3702</a>. Adds ChainMapper and ChainReducer classes allow composing
+chains of Maps and Reduces in a single Map/Reduce job, something like
+MAP+ / REDUCE MAP*.<br />(Alejandro Abdelnur via ddas)</li>
+      <li><a href="http://issues.apache.org/jira/browse/HADOOP-3445">HADOOP-3445</a>. Add capacity scheduler that provides guaranteed capacities to
+queues as a percentage of the cluster.<br />(Vivek Ratan via omalley)</li>
     </ol>
   </li>
   <li><a href="javascript:toggleList('trunk_(unreleased_changes)_._improvements_')">  IMPROVEMENTS
-</a>&nbsp;&nbsp;&nbsp;(48)
+</a>&nbsp;&nbsp;&nbsp;(49)
     <ol id="trunk_(unreleased_changes)_._improvements_">
       <li><a href="http://issues.apache.org/jira/browse/HADOOP-3908">HADOOP-3908</a>. Fuse-dfs: better error message if llibhdfs.so doesn't exist.<br />(Pete Wyckoff through zshao)</li>
       <li><a href="http://issues.apache.org/jira/browse/HADOOP-3732">HADOOP-3732</a>. Delay intialization of datanode block verification till
@@ -219,7 +229,6 @@
       <li><a href="http://issues.apache.org/jira/browse/HADOOP-3844">HADOOP-3844</a>. Include message of local exception in RPC client failures.<br />(Steve Loughran via omalley)</li>
       <li><a href="http://issues.apache.org/jira/browse/HADOOP-3935">HADOOP-3935</a>. Split out inner classes from DataNode.java.<br />(johan)</li>
       <li><a href="http://issues.apache.org/jira/browse/HADOOP-3905">HADOOP-3905</a>. Create generic interfaces for edit log streams.<br />(shv)</li>
-      <li><a href="http://issues.apache.org/jira/browse/HADOOP-3934">HADOOP-3934</a>. Upgrade log4j to 1.2.15.<br />(omalley)</li>
       <li><a href="http://issues.apache.org/jira/browse/HADOOP-3062">HADOOP-3062</a>. Add metrics to DataNode and TaskTracker to record network
 traffic for HDFS reads/writes and MR shuffling.<br />(cdouglas)</li>
       <li><a href="http://issues.apache.org/jira/browse/HADOOP-3742">HADOOP-3742</a>. Remove HDFS from public java doc and add javadoc-dev for
@@ -238,10 +247,13 @@
 NetworkTopology.pseudoSortByDistance.<br />(hairong via omalley)</li>
       <li><a href="http://issues.apache.org/jira/browse/HADOOP-3498">HADOOP-3498</a>. File globbing alternation should be able to span path
 components.<br />(tomwhite)</li>
+      <li><a href="http://issues.apache.org/jira/browse/HADOOP-3361">HADOOP-3361</a>. Implement renames for NativeS3FileSystem.<br />(Albert Chern via tomwhite)</li>
+      <li><a href="http://issues.apache.org/jira/browse/HADOOP-3605">HADOOP-3605</a>. Make EC2 scripts show an error message if AWS_ACCOUNT_ID is
+unset.<br />(Al Hoang via tomwhite)</li>
     </ol>
   </li>
   <li><a href="javascript:toggleList('trunk_(unreleased_changes)_._optimizations_')">  OPTIMIZATIONS
-</a>&nbsp;&nbsp;&nbsp;(7)
+</a>&nbsp;&nbsp;&nbsp;(8)
     <ol id="trunk_(unreleased_changes)_._optimizations_">
       <li><a href="http://issues.apache.org/jira/browse/HADOOP-3556">HADOOP-3556</a>. Removed lock contention in MD5Hash by changing the
 singleton MessageDigester by an instance per Thread using
@@ -258,10 +270,12 @@
 versions.<br />(acmurthy via omalley)</li>
       <li><a href="http://issues.apache.org/jira/browse/HADOOP-3769">HADOOP-3769</a>. Make the SampleMapper and SampleReducer from
 GenericMRLoadGenerator public, so they can be used in other contexts.<br />(Lingyun Yang via omalley)</li>
+      <li><a href="http://issues.apache.org/jira/browse/HADOOP-3514">HADOOP-3514</a>. Inline the CRCs in intermediate files as opposed to reading
+it from a different .crc file.<br />(Jothi Padmanabhan via ddas)</li>
     </ol>
   </li>
   <li><a href="javascript:toggleList('trunk_(unreleased_changes)_._bug_fixes_')">  BUG FIXES
-</a>&nbsp;&nbsp;&nbsp;(55)
+</a>&nbsp;&nbsp;&nbsp;(60)
     <ol id="trunk_(unreleased_changes)_._bug_fixes_">
       <li><a href="http://issues.apache.org/jira/browse/HADOOP-3563">HADOOP-3563</a>.  Refactor the distributed upgrade code so that it is
 easier to identify datanode and namenode related code.<br />(dhruba)</li>
@@ -320,8 +334,6 @@
 </li>
       <li><a href="http://issues.apache.org/jira/browse/HADOOP-3773">HADOOP-3773</a>. Change Pipes to set the default map output key and value
 types correctly.<br />(Koji Noguchi via omalley)</li>
-      <li><a href="http://issues.apache.org/jira/browse/HADOOP-3940">HADOOP-3940</a>. Fix in-memory merge condition to wait when there are no map
-outputs or when the final map outputs are being fetched without contention.<br />(cdouglas)</li>
       <li><a href="http://issues.apache.org/jira/browse/HADOOP-3952">HADOOP-3952</a>. Fix compilation error in TestDataJoin referencing dfs package.<br />(omalley)</li>
       <li><a href="http://issues.apache.org/jira/browse/HADOOP-3951">HADOOP-3951</a>. Fix package name for FSNamesystem logs and modify other
 hard-coded Logs to use the class name.<br />(cdouglas)</li>
@@ -369,18 +381,35 @@
 </li>
       <li><a href="http://issues.apache.org/jira/browse/HADOOP-3954">HADOOP-3954</a>. Disable record skipping by default.<br />(Sharad Agarwal via
 cdouglas)</li>
-      <li><a href="http://issues.apache.org/jira/browse/HADOOP-4050">HADOOP-4050</a>. Fix TestFailScheduler to use absolute paths for the work
+      <li><a href="http://issues.apache.org/jira/browse/HADOOP-4050">HADOOP-4050</a>. Fix TestFairScheduler to use absolute paths for the work
 directory.<br />(Matei Zaharia via omalley)</li>
       <li><a href="http://issues.apache.org/jira/browse/HADOOP-4069">HADOOP-4069</a>. Keep temporary test files from TestKosmosFileSystem under
 test.build.data instead of /tmp.<br />(lohit via omalley)</li>
+      <li><a href="http://issues.apache.org/jira/browse/HADOOP-4078">HADOOP-4078</a>. Create test files for TestKosmosFileSystem in separate
+directory under test.build.data.<br />(lohit)</li>
+      <li><a href="http://issues.apache.org/jira/browse/HADOOP-3968">HADOOP-3968</a>. Fix getFileBlockLocations calls to use FileStatus instead
+of Path reflecting the new API.<br />(Pete Wyckoff via lohit)</li>
+      <li><a href="http://issues.apache.org/jira/browse/HADOOP-3963">HADOOP-3963</a>. libhdfs does not exit on its own, instead it returns error
+to the caller and behaves as a true library.<br />(Pete Wyckoff via dhruba)</li>
+      <li><a href="http://issues.apache.org/jira/browse/HADOOP-4100">HADOOP-4100</a>. Removes the cleanupTask scheduling from the Scheduler
+implementations and moves it to the JobTracker.<br />(Amareshwari Sriramadasu via ddas)</li>
+      <li><a href="http://issues.apache.org/jira/browse/HADOOP-4097">HADOOP-4097</a>. Make hive work well with speculative execution turned on.<br />(Joydeep Sen Sarma via dhruba)</li>
+      <li><a href="http://issues.apache.org/jira/browse/HADOOP-4113">HADOOP-4113</a>. Changes to libhdfs to not exit on its own, rather return
+an error code to the caller.<br />(Pete Wyckoff via dhruba)</li>
     </ol>
   </li>
 </ul>
 <h2><a href="javascript:toggleList('release_0.18.1_-_unreleased_')">Release 0.18.1 - Unreleased
 </a></h2>
 <ul id="release_0.18.1_-_unreleased_">
+  <li><a href="javascript:toggleList('release_0.18.1_-_unreleased_._improvements_')">  IMPROVEMENTS
+</a>&nbsp;&nbsp;&nbsp;(1)
+    <ol id="release_0.18.1_-_unreleased_._improvements_">
+      <li><a href="http://issues.apache.org/jira/browse/HADOOP-3934">HADOOP-3934</a>. Upgrade log4j to 1.2.15.<br />(omalley)</li>
+    </ol>
+  </li>
   <li><a href="javascript:toggleList('release_0.18.1_-_unreleased_._bug_fixes_')">  BUG FIXES
-</a>&nbsp;&nbsp;&nbsp;(4)
+</a>&nbsp;&nbsp;&nbsp;(5)
     <ol id="release_0.18.1_-_unreleased_._bug_fixes_">
       <li><a href="http://issues.apache.org/jira/browse/HADOOP-3995">HADOOP-3995</a>. In case of quota failure on HDFS, rename does not restore
 source filename.<br />(rangadi)</li>
@@ -391,6 +420,8 @@
 collected.<br />(ddas via omalley)</li>
       <li><a href="http://issues.apache.org/jira/browse/HADOOP-4046">HADOOP-4046</a>. Made WritableComparable's constructor protected instead of
 private to re-enable class derivation.<br />(cdouglas via omalley)</li>
+      <li><a href="http://issues.apache.org/jira/browse/HADOOP-3940">HADOOP-3940</a>. Fix in-memory merge condition to wait when there are no map
+outputs or when the final map outputs are being fetched without contention.<br />(cdouglas)</li>
     </ol>
   </li>
 </ul>

Modified: hadoop/core/trunk/docs/cluster_setup.html
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/cluster_setup.html?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/docs/cluster_setup.html (original)
+++ hadoop/core/trunk/docs/cluster_setup.html Thu Sep 11 13:23:09 2008
@@ -132,6 +132,9 @@
 <a href="hdfs_shell.html">FS Shell Guide</a>
 </div>
 <div class="menuitem">
+<a href="SLG_user_guide.html">Synthetic Load Generator User Guide</a>
+</div>
+<div class="menuitem">
 <a href="distcp.html">DistCp Guide</a>
 </div>
 <div class="menuitem">

Modified: hadoop/core/trunk/docs/commands_manual.html
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/commands_manual.html?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/docs/commands_manual.html (original)
+++ hadoop/core/trunk/docs/commands_manual.html Thu Sep 11 13:23:09 2008
@@ -132,6 +132,9 @@
 <a href="hdfs_shell.html">FS Shell Guide</a>
 </div>
 <div class="menuitem">
+<a href="SLG_user_guide.html">Synthetic Load Generator User Guide</a>
+</div>
+<div class="menuitem">
 <a href="distcp.html">DistCp Guide</a>
 </div>
 <div class="menuitem">

Modified: hadoop/core/trunk/docs/distcp.html
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/distcp.html?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/docs/distcp.html (original)
+++ hadoop/core/trunk/docs/distcp.html Thu Sep 11 13:23:09 2008
@@ -131,6 +131,9 @@
 <div class="menuitem">
 <a href="hdfs_shell.html">FS Shell Guide</a>
 </div>
+<div class="menuitem">
+<a href="SLG_user_guide.html">Synthetic Load Generator User Guide</a>
+</div>
 <div class="menupage">
 <div class="menupagetitle">DistCp Guide</div>
 </div>

Modified: hadoop/core/trunk/docs/hadoop-default.html
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/hadoop-default.html?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/docs/hadoop-default.html (original)
+++ hadoop/core/trunk/docs/hadoop-default.html Thu Sep 11 13:23:09 2008
@@ -352,6 +352,12 @@
 <td><a name="dfs.replication.interval">dfs.replication.interval</a></td><td>3</td><td>The periodicity in seconds with which the namenode computes repliaction work for datanodes. </td>
 </tr>
 <tr>
+<td><a name="dfs.access.time.precision">dfs.access.time.precision</a></td><td>3600000</td><td>The access time for HDFS file is precise upto this value. 
+               The default value is 1 hour. Setting a value of 0 disables
+               access times for HDFS.
+  </td>
+</tr>
+<tr>
 <td><a name="fs.s3.block.size">fs.s3.block.size</a></td><td>67108864</td><td>Block size to use when writing files to S3.</td>
 </tr>
 <tr>
@@ -438,6 +444,15 @@
   </td>
 </tr>
 <tr>
+<td><a name="mapred.tasktracker.taskmemorymanager.monitoring-interval">mapred.tasktracker.taskmemorymanager.monitoring-interval</a></td><td>5000</td><td>The interval, in milliseconds, for which the tasktracker waits
+   between two cycles of monitoring its tasks' memory usage.</td>
+</tr>
+<tr>
+<td><a name="mapred.tasktracker.procfsbasedprocesstree.sleeptime-before-sigkill">mapred.tasktracker.procfsbasedprocesstree.sleeptime-before-sigkill</a></td><td>5000</td><td>The time, in milliseconds, the tasktracker waits for sending a
+  SIGKILL to a process that has overrun memory limits, after it has been sent
+  a SIGTERM.</td>
+</tr>
+<tr>
 <td><a name="mapred.map.tasks">mapred.map.tasks</a></td><td>2</td><td>The default number of map tasks per job.  Typically set
   to a prime several times greater than number of available hosts.
   Ignored when mapred.job.tracker is "local".  

Modified: hadoop/core/trunk/docs/hadoop_archives.html
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/hadoop_archives.html?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/docs/hadoop_archives.html (original)
+++ hadoop/core/trunk/docs/hadoop_archives.html Thu Sep 11 13:23:09 2008
@@ -132,6 +132,9 @@
 <a href="hdfs_shell.html">FS Shell Guide</a>
 </div>
 <div class="menuitem">
+<a href="SLG_user_guide.html">Synthetic Load Generator User Guide</a>
+</div>
+<div class="menuitem">
 <a href="distcp.html">DistCp Guide</a>
 </div>
 <div class="menuitem">

Modified: hadoop/core/trunk/docs/hdfs_design.html
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/hdfs_design.html?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/docs/hdfs_design.html (original)
+++ hadoop/core/trunk/docs/hdfs_design.html Thu Sep 11 13:23:09 2008
@@ -134,6 +134,9 @@
 <a href="hdfs_shell.html">FS Shell Guide</a>
 </div>
 <div class="menuitem">
+<a href="SLG_user_guide.html">Synthetic Load Generator User Guide</a>
+</div>
+<div class="menuitem">
 <a href="distcp.html">DistCp Guide</a>
 </div>
 <div class="menuitem">

Modified: hadoop/core/trunk/docs/hdfs_permissions_guide.html
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/hdfs_permissions_guide.html?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/docs/hdfs_permissions_guide.html (original)
+++ hadoop/core/trunk/docs/hdfs_permissions_guide.html Thu Sep 11 13:23:09 2008
@@ -134,6 +134,9 @@
 <a href="hdfs_shell.html">FS Shell Guide</a>
 </div>
 <div class="menuitem">
+<a href="SLG_user_guide.html">Synthetic Load Generator User Guide</a>
+</div>
+<div class="menuitem">
 <a href="distcp.html">DistCp Guide</a>
 </div>
 <div class="menuitem">

Modified: hadoop/core/trunk/docs/hdfs_quota_admin_guide.html
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/hdfs_quota_admin_guide.html?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/docs/hdfs_quota_admin_guide.html (original)
+++ hadoop/core/trunk/docs/hdfs_quota_admin_guide.html Thu Sep 11 13:23:09 2008
@@ -134,6 +134,9 @@
 <a href="hdfs_shell.html">FS Shell Guide</a>
 </div>
 <div class="menuitem">
+<a href="SLG_user_guide.html">Synthetic Load Generator User Guide</a>
+</div>
+<div class="menuitem">
 <a href="distcp.html">DistCp Guide</a>
 </div>
 <div class="menuitem">

Modified: hadoop/core/trunk/docs/hdfs_shell.html
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/hdfs_shell.html?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/docs/hdfs_shell.html (original)
+++ hadoop/core/trunk/docs/hdfs_shell.html Thu Sep 11 13:23:09 2008
@@ -132,6 +132,9 @@
 <div class="menupagetitle">FS Shell Guide</div>
 </div>
 <div class="menuitem">
+<a href="SLG_user_guide.html">Synthetic Load Generator User Guide</a>
+</div>
+<div class="menuitem">
 <a href="distcp.html">DistCp Guide</a>
 </div>
 <div class="menuitem">

Modified: hadoop/core/trunk/docs/hdfs_user_guide.html
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/hdfs_user_guide.html?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/docs/hdfs_user_guide.html (original)
+++ hadoop/core/trunk/docs/hdfs_user_guide.html Thu Sep 11 13:23:09 2008
@@ -134,6 +134,9 @@
 <a href="hdfs_shell.html">FS Shell Guide</a>
 </div>
 <div class="menuitem">
+<a href="SLG_user_guide.html">Synthetic Load Generator User Guide</a>
+</div>
+<div class="menuitem">
 <a href="distcp.html">DistCp Guide</a>
 </div>
 <div class="menuitem">

Modified: hadoop/core/trunk/docs/hod.html
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/hod.html?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/docs/hod.html (original)
+++ hadoop/core/trunk/docs/hod.html Thu Sep 11 13:23:09 2008
@@ -134,6 +134,9 @@
 <a href="hdfs_shell.html">FS Shell Guide</a>
 </div>
 <div class="menuitem">
+<a href="SLG_user_guide.html">Synthetic Load Generator User Guide</a>
+</div>
+<div class="menuitem">
 <a href="distcp.html">DistCp Guide</a>
 </div>
 <div class="menuitem">

Modified: hadoop/core/trunk/docs/hod_admin_guide.html
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/hod_admin_guide.html?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/docs/hod_admin_guide.html (original)
+++ hadoop/core/trunk/docs/hod_admin_guide.html Thu Sep 11 13:23:09 2008
@@ -134,6 +134,9 @@
 <a href="hdfs_shell.html">FS Shell Guide</a>
 </div>
 <div class="menuitem">
+<a href="SLG_user_guide.html">Synthetic Load Generator User Guide</a>
+</div>
+<div class="menuitem">
 <a href="distcp.html">DistCp Guide</a>
 </div>
 <div class="menuitem">

Modified: hadoop/core/trunk/docs/hod_config_guide.html
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/hod_config_guide.html?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/docs/hod_config_guide.html (original)
+++ hadoop/core/trunk/docs/hod_config_guide.html Thu Sep 11 13:23:09 2008
@@ -134,6 +134,9 @@
 <a href="hdfs_shell.html">FS Shell Guide</a>
 </div>
 <div class="menuitem">
+<a href="SLG_user_guide.html">Synthetic Load Generator User Guide</a>
+</div>
+<div class="menuitem">
 <a href="distcp.html">DistCp Guide</a>
 </div>
 <div class="menuitem">

Modified: hadoop/core/trunk/docs/hod_user_guide.html
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/hod_user_guide.html?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/docs/hod_user_guide.html (original)
+++ hadoop/core/trunk/docs/hod_user_guide.html Thu Sep 11 13:23:09 2008
@@ -134,6 +134,9 @@
 <a href="hdfs_shell.html">FS Shell Guide</a>
 </div>
 <div class="menuitem">
+<a href="SLG_user_guide.html">Synthetic Load Generator User Guide</a>
+</div>
+<div class="menuitem">
 <a href="distcp.html">DistCp Guide</a>
 </div>
 <div class="menuitem">

Modified: hadoop/core/trunk/docs/index.html
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/index.html?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/docs/index.html (original)
+++ hadoop/core/trunk/docs/index.html Thu Sep 11 13:23:09 2008
@@ -132,6 +132,9 @@
 <a href="hdfs_shell.html">FS Shell Guide</a>
 </div>
 <div class="menuitem">
+<a href="SLG_user_guide.html">Synthetic Load Generator User Guide</a>
+</div>
+<div class="menuitem">
 <a href="distcp.html">DistCp Guide</a>
 </div>
 <div class="menuitem">

Modified: hadoop/core/trunk/docs/linkmap.html
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/linkmap.html?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/docs/linkmap.html (original)
+++ hadoop/core/trunk/docs/linkmap.html Thu Sep 11 13:23:09 2008
@@ -132,6 +132,9 @@
 <a href="hdfs_shell.html">FS Shell Guide</a>
 </div>
 <div class="menuitem">
+<a href="SLG_user_guide.html">Synthetic Load Generator User Guide</a>
+</div>
+<div class="menuitem">
 <a href="distcp.html">DistCp Guide</a>
 </div>
 <div class="menuitem">
@@ -260,6 +263,12 @@
     
 <ul>
 <li>
+<a href="SLG_user_guide.html">Synthetic Load Generator User Guide</a>&nbsp;&nbsp;___________________&nbsp;&nbsp;<em>fs</em>
+</li>
+</ul>
+    
+<ul>
+<li>
 <a href="distcp.html">DistCp Guide</a>&nbsp;&nbsp;___________________&nbsp;&nbsp;<em>distcp</em>
 </li>
 </ul>

Modified: hadoop/core/trunk/docs/linkmap.pdf
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/linkmap.pdf?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/docs/linkmap.pdf (original)
+++ hadoop/core/trunk/docs/linkmap.pdf Thu Sep 11 13:23:09 2008
@@ -5,10 +5,10 @@
 /Producer (FOP 0.20.5) >>
 endobj
 5 0 obj
-<< /Length 1173 /Filter [ /ASCII85Decode /FlateDecode ]
+<< /Length 1192 /Filter [ /ASCII85Decode /FlateDecode ]
  >>
 stream
-Gatn'968Q9&BF88'L%gML%Vg$$n:9m'IX:*!>n;Y9!,'7W@b$c[n!l!//<go:>;1;P/mJ"Bsk",ojY7PIYe.p5T"=GMm35J0#HOY!J=5K_BA*`(^U4TENgmEY@hXM`3\AA7Bq;^p.eYt-/X[a)ud1JT%D`%]rES/.b-]j^'C.:>+#R+).c<u7M0#O0liag[_3"$5<6()Hu(tXR#[:cD3^$aPLeiu&BYo*V;:03Ago0%W`;c[Wicrl^^`No1]^2G_#=cdqObmHY]gId+Wu8HqjM/?h9Ue0KCFr:Igu\9?Ns@16'<ABUm[u"!jNeu5_)0+q]%pYq27<$9"f+Iah!q!L']bO[SU#&rsK%Pq>lKG?OtkG]kI+;gs;jl(gtEiNV[%)MC_.<p5iR(1;.N>Dq'.mC-6+/gf\5:kX^ZQ/:'^n._8Z;Adn1::u1KB_`I.1ml?9-YrDp/a;Cn;+CAJ\Y"-??PNFohLFer)RO!EGfN$2&m;rmP6G<3REdhnC^V8uDFoj8sN&"K438?]sO2CiDRSpSrcKt7]W!V$.KsAg?*-K?8q;bF\GI?EGn1lM*-1rn8<W,nR-P6Ft`@f\<o>#A,cb-h?Fe,eiYB\Uf@Q('3$"N%2V?`54;AOW.ch@)o(Idjn;&pq_'B3aKj&M9;99`c[eaCNZO`i0p"Z*9B:@Dh?#l]<@?LckI-]rPNFl_1s3ah%ubMVc%`7<D9TgiSAX>':DgO!5mpD2rN>k66k-\"%eOeSBUN[V\HgjqEt*$rt$?Ye0tXuD!&;p'!M:l\(.>+K-O_;d+\O.81o[/G@o@X[7S&@n8-@!%9*T7+?um1JT<C2@75dN"Ka"tVXU[X)7%Z07R"1McgVNQ!gr.d&EJ)U;TJ(uNGL3_6oooL7WZfg'sl]@QEUEt^YiU#$Ko>[F%Wiu$\E]4i0h8ps9N`FWK+ortj2CAn9(iD<c!`PP`IJu"Mf+$PNJnCH;(hq..H]T(#%e_qo7U8X$
 u.8Cl['>^j"X.36M-VuS3b)EiVkoZD4CG.W)]A^2+q"08K"<nbG5s5@,#)rnsMsJ??rg!TLkMY<b+\!';eg1Fb5BReP&e&GaKN@d6=!MX(2-qAP6PPPDUsGFZX[@!^3@oA:[JT99X6t7CNEk%?PmuWW^j=6bcZBBYf'3WkY2>-GN)T`cL]n7U~>
+Gatn'9lJcG&;KZQ'fr4IE;?JoURQ)R>r6CsHa/@n@_kZY9LKL6M/psm^Qb*hU^L4Xli]jM$C^_JE/#A&m<\<W*JlHR0NrB1YlC%C@.^ro&N%6Nb5/B169sNe>$'*9="C8hS*1b<WHiOW6.SgHhi._RriL:6X3K;2;7.1IDVmk^7dE9u15c^Pb+9SG&eIEFjjmYhp*NoAjN;?2Z2%""4''e<oi@$B>ZJC6BP4-+DNED'USJp0<]=<e]1.;Q3sr3u$tSTQqcAa00.)TbmhQ[+5_%#O4YN&bkJH."r85gb#D;PB#'#5[Q=8[HSRWK<a#sPo!(.:o)Bh3<>EsDt=Z<L[Wop)g8h?D]<2f^r_1:NoJ78%.JG*\1R_9CgZ`+cB"NA*03Z+D3K,CBbZA23;:T`c=T$PEXa'=mBa6WpL:!&bk7s$\$f4r]l!PT2XRiQ[jKO+%0:U.;CV]]J17d:THqqJa58@WL,Xbl*^$YP*()2N"lX)<j?F:_?mK)+AHrmeT/IFlU#$sX-K(Am*hiV#ik'(0D[<3g@8B"&SNVSp+M;kN#*C)@3IaesUOPi`JFZD_:cbq@8K\s2r'&d*_aTB`4tO>I8VSuO8]c8b1b]^C#)S'nr;#kg.IK?+a@S-IHgp%N2Ob48G-SRd7Il>*$O<LO(iE:pn&-bYiAaS8&@k;7Z9C2jq-(JIeCS=c3;oajoedZ>r0;qH]oc0f%)a!ODf?a)3kq=GFs&.E144l9Op&Oi$J<ReT9DO0HRFMCCG?e6!:N@6,VEJB(5pX.JDH\_2hCAeDTq5rV8'^]^UbQ'BW*lP&9Aq<:*\C-ApAq,.iVip9<#Z%6:X\Q-h+t7d1*BA+!V>'S085'Ui&GA!W_<;R^U#BMMM"FV/[VnTGC]Be'b:L!#^/ZO33hjI7>(6Igi>_(C\VlN(1=?ir4#olf1)A*1^ZCZL%Fji4^@di/g_>f0O+V)qo&37^fD((b:XcNnQk]:"Q8W"Z>>)$
 [044j:o?pPgXi2_[Op:O:M^SaISc/&QdV%WX6AH(#:U(LCD2f%orOuT#%sJE!LGFGf86A7bE<n7V)A-g%%u[pqSHLjs_<[X[MA+.&.ga^T(r\&K#[iKh\l$*.NI9OHnZq+(An^O>;is+b'O0aT@_(O0gL$q>Ic68BEi6L[(qtHMo9q7eChM+4S\?es`Ej6dSGN7`',dn~>
 endstream
 endobj
 6 0 obj
@@ -20,10 +20,10 @@
 >>
 endobj
 7 0 obj
-<< /Length 316 /Filter [ /ASCII85Decode /FlateDecode ]
+<< /Length 388 /Filter [ /ASCII85Decode /FlateDecode ]
  >>
 stream
-Gaqcq4\rsL&;GE/MAt"F@peTV*be%J_uQer?uW*/7\k#p%#BCo'3Z]EVrr0rIX#AEm1)E\!Oa*Z9#?.Ua$ChGNB&DY;iV(K).'Us(&\JW:ap//)BjI/RZFdj(Gk@eol;Ps$f$I`qBkndGQ[h.mOe5SXUV&IR45-]#WVm[.&$6/OCtj:U@N*LCJ:/UjZ3VkW)f7L:Tdt!j2G`2ap6arCMKc#S2KG,;R;[^74QO]V7*)rmAf^k7m3D<ZWEB=,%OACA5@Z+]@O[AGoqq=j1<oS(VZ?5McFHSm&[?;(qoeaqqcPSr^`s%qZFM"B>a~>
+Gar?-5uWCi&;BTNMESBQA$TX/E,7W0i-<Tt2sYQp)^o2(Q(j<=h>jY\L"#d(p[-_+VpXmp5To*KAeBe\'#=Gr5c@^_UmU-SdN^kMeqc/:)^1%7:QsJji!Fa4ltr0&ZPSFA_WLkSKbm;K]qbFH]Xl(>Qr7tk\%VLJPT^kj_^^Ma%*"r/bQK8@eZ*:f4."MLd0a=Q:m>lgHQ$UtI6EPhWj33hWj6VUF#C^*U\AXh8K1EDbqt"<U2u%l[=^#VMWX:i`SWDlNu+9&'$6*H9_iAnYs/FI"i&j1VpggGWt;O(&K'a7Mn[`93doUK^2LT3VHs0>8IU9BRt9#$)dmipgQ5K:epQ.3:mTX%\[,"-Pq4?L:Z"P&I1Fo#?2a#2CtRPS]SMSi~>
 endstream
 endobj
 8 0 obj
@@ -87,19 +87,19 @@
 xref
 0 14
 0000000000 65535 f 
-0000002512 00000 n 
-0000002576 00000 n 
-0000002626 00000 n 
+0000002603 00000 n 
+0000002667 00000 n 
+0000002717 00000 n 
 0000000015 00000 n 
 0000000071 00000 n 
-0000001336 00000 n 
-0000001442 00000 n 
-0000001849 00000 n 
-0000001955 00000 n 
-0000002067 00000 n 
-0000002177 00000 n 
-0000002288 00000 n 
-0000002396 00000 n 
+0000001355 00000 n 
+0000001461 00000 n 
+0000001940 00000 n 
+0000002046 00000 n 
+0000002158 00000 n 
+0000002268 00000 n 
+0000002379 00000 n 
+0000002487 00000 n 
 trailer
 <<
 /Size 14
@@ -107,5 +107,5 @@
 /Info 4 0 R
 >>
 startxref
-2748
+2839
 %%EOF

Modified: hadoop/core/trunk/docs/mapred_tutorial.html
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/mapred_tutorial.html?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/docs/mapred_tutorial.html (original)
+++ hadoop/core/trunk/docs/mapred_tutorial.html Thu Sep 11 13:23:09 2008
@@ -132,6 +132,9 @@
 <a href="hdfs_shell.html">FS Shell Guide</a>
 </div>
 <div class="menuitem">
+<a href="SLG_user_guide.html">Synthetic Load Generator User Guide</a>
+</div>
+<div class="menuitem">
 <a href="distcp.html">DistCp Guide</a>
 </div>
 <div class="menupage">

Modified: hadoop/core/trunk/docs/native_libraries.html
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/native_libraries.html?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/docs/native_libraries.html (original)
+++ hadoop/core/trunk/docs/native_libraries.html Thu Sep 11 13:23:09 2008
@@ -132,6 +132,9 @@
 <a href="hdfs_shell.html">FS Shell Guide</a>
 </div>
 <div class="menuitem">
+<a href="SLG_user_guide.html">Synthetic Load Generator User Guide</a>
+</div>
+<div class="menuitem">
 <a href="distcp.html">DistCp Guide</a>
 </div>
 <div class="menuitem">

Modified: hadoop/core/trunk/docs/quickstart.html
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/quickstart.html?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/docs/quickstart.html (original)
+++ hadoop/core/trunk/docs/quickstart.html Thu Sep 11 13:23:09 2008
@@ -132,6 +132,9 @@
 <a href="hdfs_shell.html">FS Shell Guide</a>
 </div>
 <div class="menuitem">
+<a href="SLG_user_guide.html">Synthetic Load Generator User Guide</a>
+</div>
+<div class="menuitem">
 <a href="distcp.html">DistCp Guide</a>
 </div>
 <div class="menuitem">

Modified: hadoop/core/trunk/docs/streaming.html
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/docs/streaming.html?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/docs/streaming.html (original)
+++ hadoop/core/trunk/docs/streaming.html Thu Sep 11 13:23:09 2008
@@ -135,6 +135,9 @@
 <a href="hdfs_shell.html">FS Shell Guide</a>
 </div>
 <div class="menuitem">
+<a href="SLG_user_guide.html">Synthetic Load Generator User Guide</a>
+</div>
+<div class="menuitem">
 <a href="distcp.html">DistCp Guide</a>
 </div>
 <div class="menuitem">

Added: hadoop/core/trunk/src/docs/src/documentation/content/xdocs/SLG_user_guide.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/docs/src/documentation/content/xdocs/SLG_user_guide.xml?rev=694458&view=auto
==============================================================================
--- hadoop/core/trunk/src/docs/src/documentation/content/xdocs/SLG_user_guide.xml (added)
+++ hadoop/core/trunk/src/docs/src/documentation/content/xdocs/SLG_user_guide.xml Thu Sep 11 13:23:09 2008
@@ -0,0 +1,192 @@
+<?xml version="1.0"?>
+<!--
+  Copyright 2002-2004 The Apache Software Foundation
+  
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+  
+      http://www.apache.org/licenses/LICENSE-2.0
+      
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
+<document>
+	<header>
+		<title> Synthetic Load Generator User Guide </title>
+	</header>
+	<body>
+		<section>
+			<title> Description </title>
+			<p>
+        The synthetic load generator is a tool for testing NameNode behavior
+        under different client loads. The user can generate different mixes 
+        of read, write, and list requests by specifying the probabilities of
+        read and write. The user controls the intensity of the load by adjusting
+        parameters for the number of worker threads and the delay between 
+        operations. While load generators are running, the user can profile and
+        monitor the running of the NameNode. When a load generator exits, it
+        prints some NameNode statistics like the average execution time of each
+        kind of operation and the NameNode throughput.
+                       </p>
+                </section>
+		<section>
+			<title> Synopsis </title>
+			<p>
+        <code>java LoadGenerator [options]</code><br/>
+                        </p>
+                        <p>
+        Options include:<br/>
+        <code>&nbsp;&nbsp;-readProbability &lt;read probability&gt;</code><br/>
+        <code>&nbsp;&nbsp;&nbsp;&nbsp;the probability of the read operation;
+                default is 0.3333. </code><br/>
+        <code>&nbsp;&nbsp;-writeProbability &lt;write probability&gt;</code><br/>
+        <code>&nbsp;&nbsp;&nbsp;&nbsp;the probability of the write 
+                operations; default is 0.3333.</code><br/>
+        <code>&nbsp;&nbsp;-root &lt;test space root&gt;</code><br/>
+        <code>&nbsp;&nbsp;&nbsp;&nbsp;the root of the test space;
+                default is /testLoadSpace.</code><br/>
+        <code>&nbsp;&nbsp;-maxDelayBetweenOps 
+                &lt;maxDelayBetweenOpsInMillis&gt;</code><br/> 
+        <code>&nbsp;&nbsp;&nbsp;&nbsp;the maximum delay between two consecutive
+                operations in a thread; default is 0 indicating no delay.
+                </code><br/>
+        <code>&nbsp;&nbsp;-numOfThreads &lt;numOfThreads&gt;</code><br/> 
+        <code>&nbsp;&nbsp;&nbsp;&nbsp;the number of threads to spawn; 
+                default is 200.</code><br/>
+        <code>&nbsp;&nbsp;-elapsedTime &lt;elapsedTimeInSecs&gt;</code><br/>
+        <code>&nbsp;&nbsp;&nbsp;&nbsp;the number of seconds that the program 
+                will run; A value of zero indicates that the program runs
+                forever. The default value is 0.</code><br/>
+        <code>&nbsp;&nbsp;-startTime &lt;startTimeInMillis&gt;</code><br/> 
+        <code>&nbsp;&nbsp;&nbsp;&nbsp;the time that all worker threads 
+                start to run. By default it is 10 seconds after the main 
+                program starts running.This creates a barrier if more than
+                one load generator is running.
+        </code><br/>
+        <code>&nbsp;&nbsp;-seed &lt;seed&gt;</code><br/>
+        <code>&nbsp;&nbsp;&nbsp;&nbsp;the random generator seed for repeating 
+                requests to NameNode when running with a single thread;
+                default is the current time.</code><br/>
+			</p>
+			<p>
+        After command line argument parsing, the load generator traverses 
+        the test space and builds a table of all directories and another table
+        of all files in the test space. It then waits until the start time to
+        spawn the number of worker threads as specified by the user. Each
+        thread sends a stream of requests to NameNode. At each iteration, 
+        it first decides if it is going to read a file, create a file, or
+        list a directory following the read and write probabilities specified
+        by the user. The listing probability is equal to 
+        <em>1-read probability-write probability</em>. When reading, 
+        it randomly picks a file in the test space and reads the entire file. 
+        When writing, it randomly picks a directory in the test space and 
+        creates a file there. To avoid two threads with the same load 
+        generator or from two different load generators create the same 
+        file, the file name consists of the current machine's host name 
+        and the thread id. The length of the file follows Gaussian 
+        distribution with an average size of 2 blocks and the standard 
+        deviation of 1. The new file is filled with byte 'a'. To avoid
+        the test space to grow indefinitely, the file is deleted immediately
+        after the file creation completes. While listing, it randomly 
+        picks a directory in the test space and lists its content. 
+        After an operation completes, the thread pauses for a random 
+        amount of time in the range of [0, maxDelayBetweenOps] if the 
+        specified maximum delay is not zero. All threads are stopped when 
+        the specified elapsed time is passed. Before exiting, the program 
+        prints the average execution for each kind of NameNode operations, 
+        and the number of requests served by the NameNode per second.
+                        </p>
+                </section>
+                <section>
+                        <title> Test Space Population </title>
+                        <p>
+        The user needs to populate a test space before she runs a 
+        load generator. The structure generator generates a random 
+        test space structure and the data generator creates the files 
+        and directories of the test space in Hadoop distributed file system.
+                        </p>
+                        <section>
+                                <title> Structure Generator </title>
+                                <p>
+        This tool generates a random namespace structure with the 
+        following constraints:
+                                </p>
+                                        <ol>
+        <li>The number of subdirectories that a directory can have is 
+            a random number in [minWidth, maxWidth].</li>
+        <li>The maximum depth of each subdirectory is a random number 
+            [2*maxDepth/3, maxDepth].</li>
+        <li>Files are randomly placed in leaf directories. The size of 
+            each file follows Gaussian distribution with an average size 
+            of 1 block and a standard deviation of 1.</li>
+                                        </ol>
+                                <p>
+        The generated namespace structure is described by two files in 
+        the output directory. Each line of the first file contains the 
+        full name of a leaf directory. Each line of the second file 
+        contains the full name of a file and its size, separated by a blank.
+                                </p>
+                                <p>
+        The synopsis of the command is
+                                </p>
+                                <p>
+        <code>java StructureGenerator [options]</code>
+                                </p>
+                                <p>
+        Options include:<br/>
+        <code>&nbsp;&nbsp;-maxDepth &lt;maxDepth&gt;</code><br/>
+        <code>&nbsp;&nbsp;&nbsp;&nbsp;maximum depth of the directory tree; 
+                default is 5.</code><br/>
+        <code>&nbsp;&nbsp;-minWidth &lt;minWidth&gt;</code><br/> 
+        <code>&nbsp;&nbsp;&nbsp;&nbsp;minimum number of subdirectories per 
+                directories; default is 1.</code><br/>
+        <code>&nbsp;&nbsp;-maxWidth &lt;maxWidth&gt;</code><br/> 
+        <code>&nbsp;&nbsp;&nbsp;&nbsp;maximum number of subdirectories per 
+                directories; default is 5.</code><br/>
+        <code>&nbsp;&nbsp;-numOfFiles &lt;#OfFiles&gt;</code><br/> 
+        <code>&nbsp;&nbsp;&nbsp;&nbsp;the total number of files in the test 
+                space; default is 10.</code><br/>
+        <code>&nbsp;&nbsp;-avgFileSize &lt;avgFileSizeInBlocks&gt;</code><br/>
+        <code>&nbsp;&nbsp;&nbsp;&nbsp;average size of blocks; default is 1.
+                </code><br/>
+        <code>&nbsp;&nbsp;-outDir &lt;outDir&gt;</code><br/>
+        <code>&nbsp;&nbsp;&nbsp;&nbsp;output directory; default is the 
+                current directory. </code><br/>
+        <code>&nbsp;&nbsp;-seed &lt;seed&gt;</code><br/>
+        <code>&nbsp;&nbsp;&nbsp;&nbsp;random number generator seed; 
+                default is the current time.</code><br/>
+                                </p>
+                        </section>
+                        <section>
+                                <title> Test Space Generator </title>
+                                <p>
+        This tool reads the directory structure and file structure from 
+        the input directory and creates the namespace in Hadoop distributed
+        file system. All files are filled with byte 'a'.
+                                </p>
+                                <p>
+        The synopsis of the command is
+                                </p>
+                                <p>
+        <code>java DataGenerator [options]</code>
+                                </p>
+                                <p>
+        Options include:<br/>
+        <code>&nbsp;&nbsp;-inDir &lt;inDir&gt;</code><br/>
+        <code>&nbsp;&nbsp;&nbsp;&nbsp;input directory name where directory/file
+                structures are stored; default is the current directory.
+        </code><br/>
+        <code>&nbsp;&nbsp;-root &lt;test space root&gt;</code><br/>
+        <code>&nbsp;&nbsp;&nbsp;&nbsp;the name of the root directory which the 
+                new namespace is going to be placed under; 
+                default is "/testLoadSpace".</code><br/>
+                                </p>
+		        </section>
+                </section>
+	</body>
+</document>

Modified: hadoop/core/trunk/src/docs/src/documentation/content/xdocs/site.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/docs/src/documentation/content/xdocs/site.xml?rev=694458&r1=694457&r2=694458&view=diff
==============================================================================
--- hadoop/core/trunk/src/docs/src/documentation/content/xdocs/site.xml (original)
+++ hadoop/core/trunk/src/docs/src/documentation/content/xdocs/site.xml Thu Sep 11 13:23:09 2008
@@ -41,6 +41,7 @@
     <hdfs      label="HDFS Quotas Administrator Guide" href="hdfs_quota_admin_guide.html" />
     <commands  label="Commands Manual"     href="commands_manual.html" />
     <fs        label="FS Shell Guide"     href="hdfs_shell.html" />
+    <fs        label="Synthetic Load Generator User Guide"  href="SLG_user_guide.html" />
     <distcp    label="DistCp Guide"       href="distcp.html" />
     <mapred    label="Map-Reduce Tutorial" href="mapred_tutorial.html" />
     <mapred    label="Native Hadoop Libraries" href="native_libraries.html" />

Added: hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/DataGenerator.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/DataGenerator.java?rev=694458&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/DataGenerator.java (added)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/DataGenerator.java Thu Sep 11 13:23:09 2008
@@ -0,0 +1,160 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.loadGenerator;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * This program reads the directory structure and file structure from
+ * the input directory and creates the namespace in the file system
+ * specified by the configuration in the specified root.
+ * All the files are filled with 'a'.
+ * 
+ * The synopsis of the command is
+ * java DataGenerator 
+ *   -inDir <inDir>: input directory name where directory/file structures
+ *                   are stored. Its default value is the current directory.
+ *   -root <root>: the name of the root directory which the new namespace 
+ *                 is going to be placed under. 
+ *                 Its default value is "/testLoadSpace".
+ */
+public class DataGenerator extends Configured implements Tool {
+  private File inDir = StructureGenerator.DEFAULT_STRUCTURE_DIRECTORY;
+  private Path root = DEFAULT_ROOT;
+  private FileSystem fs;
+  final static private long BLOCK_SIZE = 10;
+  final static private String USAGE = "java DataGenerator " +
+  		"-inDir <inDir> " +
+  		"-root <root>";
+  
+  /** default name of the root where the test namespace will be placed under */
+  final static Path DEFAULT_ROOT = new Path("/testLoadSpace");
+  
+  /** Main function.
+   * It first parses the command line arguments.
+   * It then reads the directory structure from the input directory 
+   * structure file and creates directory structure in the file system
+   * namespace. Afterwards it reads the file attributes and creates files 
+   * in the file. All file content is filled with 'a'.
+   */
+  public int run(String[] args) throws Exception {
+    int exitCode = 0;
+    exitCode = init(args);
+    if (exitCode != 0) {
+      return exitCode;
+    }
+    genDirStructure();
+    genFiles();
+    return exitCode;
+  }
+
+  /** Parse the command line arguments and initialize the data */
+  private int init(String[] args) {
+    try { // initialize file system handle
+      fs = FileSystem.get(getConf());
+    } catch (IOException ioe) {
+      System.err.println("Can not initialize the file system: " + 
+          ioe.getLocalizedMessage());
+      return -1;
+    }
+
+    for (int i = 0; i < args.length; i++) { // parse command line
+      if (args[i].equals("-root")) {
+        root = new Path(args[++i]);
+      } else if (args[i].equals("-inDir")) {
+        inDir = new File(args[++i]);
+      } else {
+        System.err.println(USAGE);
+        ToolRunner.printGenericCommandUsage(System.err);
+        System.exit(-1);
+      }
+    }
+    return 0;
+  }
+  
+  /** Read directory structure file under the input directory.
+   * Create each directory under the specified root.
+   * The directory names are relative to the specified root.
+   */
+  private void genDirStructure() throws IOException {
+    BufferedReader in = new BufferedReader(
+        new FileReader(new File(inDir, 
+            StructureGenerator.DIR_STRUCTURE_FILE_NAME)));
+    String line;
+    while ((line=in.readLine()) != null) {
+      fs.mkdirs(new Path(root+line));
+    }
+  }
+
+  /** Read file structure file under the input directory.
+   * Create each file under the specified root.
+   * The file names are relative to the root.
+   */
+  private void genFiles() throws IOException {
+    BufferedReader in = new BufferedReader(
+        new FileReader(new File(inDir, 
+            StructureGenerator.FILE_STRUCTURE_FILE_NAME)));
+    String line;
+    while ((line=in.readLine()) != null) {
+      String[] tokens = line.split(" ");
+      if (tokens.length != 2) {
+        throw new IOException("Expect at most 2 tokens per line: " + line);
+      }
+      String fileName = root+tokens[0];
+      long fileSize = (long)(BLOCK_SIZE*Double.parseDouble(tokens[1]));
+      genFile(new Path(fileName), fileSize);
+    }
+  }
+  
+  /** Create a file with the name <code>file</code> and 
+   * a length of <code>fileSize</code>. The file is filled with character 'a'.
+   */
+  private void genFile(Path file, long fileSize) throws IOException {
+    FSDataOutputStream out = fs.create(file, true, 
+        getConf().getInt("io.file.buffer.size", 4096),
+        (short)getConf().getInt("dfs.replication", 3),
+        fs.getDefaultBlockSize());
+    for(long i=0; i<fileSize; i++) {
+      out.writeByte('a');
+    }
+    out.close();
+  }
+  
+  /** Main program.
+   * 
+   * @param args Command line arguments
+   * @throws Exception
+   */
+  public static void main(String[] args) throws Exception {
+    int res = ToolRunner.run(new Configuration(),
+        new DataGenerator(), args);
+    System.exit(res);
+  }
+}



Mime
View raw message