hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject svn commit: r694458 [2/2] - in /hadoop/core/trunk: ./ docs/ src/docs/src/documentation/content/xdocs/ src/test/org/apache/hadoop/fs/loadGenerator/
Date Thu, 11 Sep 2008 20:23:18 GMT
Added: hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java?rev=694458&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java (added)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java Thu Sep
11 13:23:09 2008
@@ -0,0 +1,466 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.loadGenerator;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Random;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/** The load generator is a tool for testing NameNode behavior under
+ * different client loads.
+ * It allows the user to generate different mixes of read, write,
+ * and list requests by specifying the probabilities of read and
+ * write. The user controls the the intensity of the load by
+ * adjusting parameters for the number of worker threads and the delay
+ * between operations. While load generators are running, the user
+ * can profile and monitor the running of the NameNode. When a load
+ * generator exits, it print some NameNode statistics like the average
+ * execution time of each kind of operations and the NameNode
+ * throughput.
+ * 
+ * After command line argument parsing and data initialization,
+ * the load generator spawns the number of worker threads 
+ * as specified by the user.
+ * Each thread sends a stream of requests to the NameNode.
+ * For each iteration, it first decides if it is going to read a file,
+ * create a file, or listing a directory following the read and write 
+ * probabilities specified by the user.
+ * When reading, it randomly picks a file in the test space and reads
+ * the entire file. When writing, it randomly picks a directory in the
+ * test space and creates a file whose name consists of the current 
+ * machine's host name and the thread id. The length of the file
+ * follows Gaussian distribution with an average size of 2 blocks and
+ * the standard deviation of 1 block. The new file is filled with 'a'.
+ * Immediately after the file creation completes, the file is deleted
+ * from the test space.
+ * While listing, it randomly picks a directory in the test space and
+ * list the directory content.
+ * Between two consecutive operations, the thread pauses for a random
+ * amount of time in the range of [0, maxDelayBetweenOps] 
+ * if the specified max delay is not zero.
+ * All threads are stopped when the specified elapsed time is passed.
+ * Before exiting, the program prints the average execution for 
+ * each kind of NameNode operations, and the number of requests
+ * served by the NameNode.
+ *
+ * The synopsis of the command is
+ * java LoadGenerator
+ *   -readProbability <read probability>: read probability [0, 1]
+ *                                        with a default value of 0.3333. 
+ *   -writeProbability <write probability>: write probability [0, 1]
+ *                                         with a default value of 0.3333.
+ *   -root <root>: test space with a default value of /testLoadSpace
+ *   -maxDelayBetweenOps <maxDelayBetweenOpsInMillis>: 
+ *      Max delay in the unit of milliseconds between two operations with a 
+ *      default value of 0 indicating no delay.
+ *   -numOfThreads <numOfThreads>: 
+ *      number of threads to spawn with a default value of 200.
+ *   -elapsedTime <elapsedTimeInSecs>: 
+ *      the elapsed time of program with a default value of 0 
+ *      indicating running forever
+ *   -startTime <startTimeInMillis> : when the threads start to run.
+ */
+public class LoadGenerator extends Configured implements Tool {
+  private volatile boolean shouldRun = true;
+  private Path root = DataGenerator.DEFAULT_ROOT;
+  private FileSystem fs;
+  private int maxDelayBetweenOps = 0;
+  private int numOfThreads = 200;
+  private double readPr = 0.3333;
+  private double writePr = 0.3333;
+  private long elapsedTime = 0;
+  private long startTime = System.currentTimeMillis()+10000;
+  final static private int BLOCK_SIZE = 10;
+  private ArrayList<String> files = new ArrayList<String>();  // a table of file
names
+  private ArrayList<String> dirs = new ArrayList<String>(); // a table of directory
names
+  private Random r = null;
+  final private static String USAGE = "java LoadGenerator\n" +
+  	"-readProbability <read probability>\n" +
+    "-writeProbability <write probability>\n" +
+    "-root <root>\n" +
+    "-maxDelayBetweenOps <maxDelayBetweenOpsInMillis>\n" +
+    "-numOfThreads <numOfThreads>\n" +
+    "-elapsedTime <elapsedTimeInSecs>\n" +
+    "-startTime <startTimeInMillis>";
+  final private String hostname;
+  
+  /** Constructor */
+  public LoadGenerator() throws IOException, UnknownHostException {
+    InetAddress addr = InetAddress.getLocalHost();
+    hostname = addr.getHostName();
+  }
+
+  private final static int OPEN = 0;
+  private final static int LIST = 1;
+  private final static int CREATE = 2;
+  private final static int WRITE_CLOSE = 3;
+  private final static int DELETE = 4;
+  private final static int TOTAL_OP_TYPES =5;
+  private long [] executionTime = new long[TOTAL_OP_TYPES];
+  private long [] totalNumOfOps = new long[TOTAL_OP_TYPES];
+  
+  /** A thread sends a stream of requests to the NameNode.
+   * At each iteration, it first decides if it is going to read a file,
+   * create a file, or listing a directory following the read
+   * and write probabilities.
+   * When reading, it randomly picks a file in the test space and reads
+   * the entire file. When writing, it randomly picks a directory in the
+   * test space and creates a file whose name consists of the current 
+   * machine's host name and the thread id. The length of the file
+   * follows Gaussian distribution with an average size of 2 blocks and
+   * the standard deviation of 1 block. The new file is filled with 'a'.
+   * Immediately after the file creation completes, the file is deleted
+   * from the test space.
+   * While listing, it randomly picks a directory in the test space and
+   * list the directory content.
+   * Between two consecutive operations, the thread pauses for a random
+   * amount of time in the range of [0, maxDelayBetweenOps] 
+   * if the specified max delay is not zero.
+   * A thread runs for the specified elapsed time if the time isn't zero.
+   * Otherwise, it runs forever.
+   */
+  private class DFSClientThread extends Thread {
+    private int id;
+    private long [] executionTime = new long[TOTAL_OP_TYPES];
+    private long [] totalNumOfOps = new long[TOTAL_OP_TYPES];
+    private byte[] buffer = new byte[1024];
+    
+    private DFSClientThread(int id) {
+      this.id = id;
+    }
+    
+    /** Main loop
+     * Each iteration decides what's the next operation and then pauses.
+     */
+    public void run() {
+      try {
+        while (shouldRun) {
+          nextOp();
+          delay();
+        }
+      } catch (Exception ioe) {
+        System.err.println(ioe.getLocalizedMessage());
+        ioe.printStackTrace();
+      }
+    }
+    
+    /** Let the thread pause for a random amount of time in the range of
+     * [0, maxDelayBetweenOps] if the delay is not zero. Otherwise, no pause.
+     */
+    private void delay() throws InterruptedException {
+      if (maxDelayBetweenOps>0) {
+        int delay = r.nextInt(maxDelayBetweenOps);
+        Thread.sleep(delay);
+      }
+    }
+    
+    /** Perform the next operation. 
+     * 
+     * Depending on the read and write probabilities, the next
+     * operation could be either read, write, or list.
+     */
+    private void nextOp() throws IOException {
+      double rn = r.nextDouble();
+      if (rn < readPr) {
+        read();
+      } else if (rn < readPr+writePr) {
+        write();
+      } else {
+        list();
+      }
+    }
+    
+    /** Read operation randomly picks a file in the test space and reads
+     * the entire file */
+    private void read() throws IOException {
+      String fileName = files.get(r.nextInt(files.size()));
+      long startTime = System.currentTimeMillis();
+      InputStream in = fs.open(new Path(fileName));
+      executionTime[OPEN] += (System.currentTimeMillis()-startTime);
+      totalNumOfOps[OPEN]++;
+      while (in.read(buffer) != -1) {}
+      in.close();
+    }
+    
+    /** The write operation randomly picks a directory in the
+     * test space and creates a file whose name consists of the current 
+     * machine's host name and the thread id. The length of the file
+     * follows Gaussian distribution with an average size of 2 blocks and
+     * the standard deviation of 1 block. The new file is filled with 'a'.
+     * Immediately after the file creation completes, the file is deleted
+     * from the test space.
+     */
+    private void write() throws IOException {
+      String dirName = dirs.get(r.nextInt(dirs.size()));
+      Path file = new Path(dirName, hostname+id);
+      double fileSize = 0;
+      while ((fileSize = r.nextGaussian()+2)<=0) {}
+      genFile(file, (long)(fileSize*BLOCK_SIZE));
+      long startTime = System.currentTimeMillis();
+      fs.delete(file, true);
+      executionTime[DELETE] += (System.currentTimeMillis()-startTime);
+      totalNumOfOps[DELETE]++;
+    }
+    
+    /** The list operation randomly picks a directory in the test space and
+     * list the directory content.
+     */
+    private void list() throws IOException {
+      String dirName = dirs.get(r.nextInt(dirs.size()));
+      long startTime = System.currentTimeMillis();
+      fs.listStatus(new Path(dirName));
+      executionTime[LIST] += (System.currentTimeMillis()-startTime);
+      totalNumOfOps[LIST]++;
+    }
+  }
+  
+  /** Main function:
+   * It first initializes data by parsing the command line arguments.
+   * It then starts the number of DFSClient threads as specified by
+   * the user.
+   * It stops all the threads when the specified elapsed time is passed.
+   * Before exiting, it prints the average execution for 
+   * each operation and operation throughput.
+   */
+  public int run(String[] args) throws Exception {
+    int exitCode = init(args);
+    if (exitCode != 0) {
+      return exitCode;
+    }
+    
+    barrier();
+    
+    DFSClientThread[] threads = new DFSClientThread[numOfThreads];
+    for (int i=0; i<numOfThreads; i++) {
+      threads[i] = new DFSClientThread(i); 
+      threads[i].start();
+    }
+    if (elapsedTime>0) {
+      Thread.sleep(elapsedTime*1000);
+      shouldRun = false;
+    } 
+    for (DFSClientThread thread : threads) {
+      thread.join();
+      for (int i=0; i<TOTAL_OP_TYPES; i++) {
+        executionTime[i] += thread.executionTime[i];
+        totalNumOfOps[i] += thread.totalNumOfOps[i];
+      }
+    }
+    long totalOps = 0;
+    for (int i=0; i<TOTAL_OP_TYPES; i++) {
+      totalOps += totalNumOfOps[i];
+    }
+    
+    if (totalNumOfOps[OPEN] != 0) {
+      System.out.println("Average open execution time: " + 
+          (double)executionTime[OPEN]/totalNumOfOps[OPEN] + "ms");
+    }
+    if (totalNumOfOps[LIST] != 0) {
+      System.out.println("Average list execution time: " + 
+          (double)executionTime[LIST]/totalNumOfOps[LIST] + "ms");
+    }
+    if (totalNumOfOps[DELETE] != 0) {
+      System.out.println("Average deletion execution time: " + 
+          (double)executionTime[DELETE]/totalNumOfOps[DELETE] + "ms");
+      System.out.println("Average create execution time: " + 
+          (double)executionTime[CREATE]/totalNumOfOps[CREATE] + "ms");
+      System.out.println("Average write_close execution time: " + 
+          (double)executionTime[WRITE_CLOSE]/totalNumOfOps[WRITE_CLOSE] + "ms");
+    }
+    if (elapsedTime != 0) { 
+      System.out.println("Average operations per second: " + 
+          (double)totalOps/elapsedTime +"ops/s");
+    }
+    System.out.println();
+    return exitCode;
+  }
+
+  /** Parse the command line arguments and initialize the data */
+  private int init(String[] args) throws IOException {
+    try {
+      fs = FileSystem.get(getConf());
+    } catch (IOException ioe) {
+      System.err.println("Can not initialize the file system: " + 
+          ioe.getLocalizedMessage());
+      return -1;
+    }
+    int hostHashCode = hostname.hashCode();
+    try {
+      for (int i = 0; i < args.length; i++) { // parse command line
+        if (args[i].equals("-readProbability")) {
+          readPr = Double.parseDouble(args[++i]);
+          if (readPr<0 || readPr>1) {
+            System.err.println( 
+                "The read probability must be [0, 1]: " + readPr);
+            return -1;
+          }
+        } else if (args[i].equals("-writeProbability")) {
+          writePr = Double.parseDouble(args[++i]);
+          if (writePr<0 || writePr>1) {
+            System.err.println( 
+                "The write probability must be [0, 1]: " + writePr);
+            return -1;
+          }
+        } else if (args[i].equals("-root")) {
+          root = new Path(args[++i]);
+        } else if (args[i].equals("-maxDelayBetweenOps")) {
+          maxDelayBetweenOps = Integer.parseInt(args[++i]); // in milliseconds
+        } else if (args[i].equals("-numOfThreads")) {
+          numOfThreads = Integer.parseInt(args[++i]);
+          if (numOfThreads <= 0) {
+            System.err.println(
+                "Number of threads must be positive: " + numOfThreads);
+            return -1;
+          }
+        } else if (args[i].equals("-startTime")) {
+          startTime = Long.parseLong(args[++i]);
+        } else if (args[i].equals("-elapsedTime")) {
+          elapsedTime = Long.parseLong(args[++i]);
+        } else if (args[i].equals("-seed")) {
+          r = new Random(Long.parseLong(args[++i])+hostHashCode);
+        } else {
+          System.err.println(USAGE);
+          ToolRunner.printGenericCommandUsage(System.err);
+          return -1;
+        }
+      }
+    } catch (NumberFormatException e) {
+      System.err.println("Illegal parameter: " + e.getLocalizedMessage());
+      System.err.println(USAGE);
+      return -1;
+    }
+
+    if (readPr+writePr <0 || readPr+writePr>1) {
+      System.err.println(
+          "The sum of read probability and write probability must be [0, 1]: " +
+          readPr + " "+writePr);
+      return -1;
+    }
+    
+    if (r==null) {
+      r = new Random(System.currentTimeMillis()+hostHashCode);
+    }
+    
+    return initFileDirTables();
+  }
+  
+  /** Create a table that contains all directories under root and
+   * another table that contains all files under root.
+   */
+  private int initFileDirTables() {
+    try {
+      initFileDirTables(root);
+    } catch (IOException e) {
+      System.err.println(e.getLocalizedMessage());
+      e.printStackTrace();
+      return -1;
+    }
+    if (dirs.isEmpty()) {
+      System.err.println("The test space " + root + " is empty");
+      return -1;
+    }
+    if (files.isEmpty()) {
+      System.err.println("The test space " + root + 
+          " does not have any file");
+      return -1;
+    }
+    return 0;
+  }
+  
+  /** Create a table that contains all directories under the specified path and
+   * another table that contains all files under the specified path and
+   * whose name starts with "_file_".
+   */
+  private void initFileDirTables(Path path) throws IOException {
+    FileStatus[] stats = fs.listStatus(path);
+    if (stats != null) { 
+      for (FileStatus stat : stats) {
+        if (stat.isDir()) {
+          dirs.add(stat.getPath().toString());
+          initFileDirTables(stat.getPath());
+        } else {
+          Path filePath = stat.getPath();
+          if (filePath.getName().startsWith(StructureGenerator.FILE_NAME_PREFIX)) {
+            files.add(filePath.toString());
+          }
+        }
+      }
+    }
+  }
+  
+  /** Returns when the current number of seconds from the epoch equals
+   * the command line argument given by <code>-startTime</code>.
+   * This allows multiple instances of this program, running on clock
+   * synchronized nodes, to start at roughly the same time.
+   */
+  private void barrier() {
+    long sleepTime;
+    while ((sleepTime = startTime - System.currentTimeMillis()) > 0) {
+      try {
+        Thread.sleep(sleepTime);
+      } catch (InterruptedException ex) {
+      }
+    }
+  }
+
+  /** Create a file with a length of <code>fileSize</code>.
+   * The file is filled with 'a'.
+   */
+  private void genFile(Path file, long fileSize) throws IOException {
+    long startTime = System.currentTimeMillis();
+    FSDataOutputStream out = fs.create(file, true, 
+        getConf().getInt("io.file.buffer.size", 4096),
+        (short)getConf().getInt("dfs.replication", 3),
+        fs.getDefaultBlockSize());
+    executionTime[CREATE] += (System.currentTimeMillis()-startTime);
+    totalNumOfOps[CREATE]++;
+
+    for (long i=0; i<fileSize; i++) {
+      out.writeByte('a');
+    }
+    startTime = System.currentTimeMillis();
+    out.close();
+    executionTime[WRITE_CLOSE] += (System.currentTimeMillis()-startTime);
+    totalNumOfOps[WRITE_CLOSE]++;
+  }
+  
+  /** Main program
+   * 
+   * @param args command line arguments
+   * @throws Exception
+   */
+  public static void main(String[] args) throws Exception {
+    int res = ToolRunner.run(new Configuration(),
+        new LoadGenerator(), args);
+    System.exit(res);
+  }
+
+}

Added: hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/StructureGenerator.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/StructureGenerator.java?rev=694458&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/StructureGenerator.java
(added)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/StructureGenerator.java
Thu Sep 11 13:23:09 2008
@@ -0,0 +1,307 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.loadGenerator;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * This program generates a random namespace structure with the following
+ * constraints:
+ * 1. The number of subdirectories is a random number in [minWidth, maxWidth].
+ * 2. The maximum depth of each subdirectory is a random number 
+ *    [2*maxDepth/3, maxDepth].
+ * 3. Files are randomly placed in the empty directories. The size of each
+ *    file follows Gaussian distribution.
+ * The generated namespace structure is described by two files in the output
+ * directory. Each line of the first file 
+ * contains the full name of a leaf directory.  
+ * Each line of the second file contains
+ * the full name of a file and its size, separated by a blank.
+ * 
+ * The synopsis of the command is
+ * java StructureGenerator
+    -maxDepth <maxDepth> : maximum depth of the directory tree; default is 5.
+    -minWidth <minWidth> : minimum number of subdirectories per directories; default
is 1
+    -maxWidth <maxWidth> : maximum number of subdirectories per directories; default
is 5
+    -numOfFiles <#OfFiles> : the total number of files; default is 10.
+    -avgFileSize <avgFileSizeInBlocks>: average size of blocks; default is 1.
+    -outDir <outDir>: output directory; default is the current directory.
+    -seed <seed>: random number generator seed; default is the current time.
+ */
+public class StructureGenerator {
+  private int maxDepth = 5;
+  private int minWidth = 1;
+  private int maxWidth = 5;
+  private int numOfFiles = 10;
+  private double avgFileSize = 1;
+  private File outDir = DEFAULT_STRUCTURE_DIRECTORY;
+  final static private String USAGE = "java StructureGenerator\n" +
+  	"-maxDepth <maxDepth>\n" +
+    "-minWidth <minWidth>\n" +
+    "-maxWidth <maxWidth>\n" +
+    "-numOfFiles <#OfFiles>\n" +
+    "-avgFileSize <avgFileSizeInBlocks>\n" +
+    "-outDir <outDir>\n" +
+    "-seed <seed>";
+  
+  private Random r = null; 
+  
+  /** Default directory for storing file/directory structure */
+  final static File DEFAULT_STRUCTURE_DIRECTORY = new File(".");
+  /** The name of the file for storing directory structure */
+  final static String DIR_STRUCTURE_FILE_NAME = "dirStructure";
+  /** The name of the file for storing file structure */
+  final static String FILE_STRUCTURE_FILE_NAME = "fileStructure";
+  /** The name prefix for the files created by this program */
+  final static String FILE_NAME_PREFIX = "_file_";
+  
+  /**
+   * The main function first parses the command line arguments,
+   * then generates in-memory directory structure and outputs to a file,
+   * last generates in-memory files and outputs them to a file.
+   */
+  public int run(String[] args) throws Exception {
+    int exitCode = 0;
+    exitCode = init(args);
+    if (exitCode != 0) {
+      return exitCode;
+    }
+    genDirStructure();
+    output(new File(outDir, DIR_STRUCTURE_FILE_NAME));
+    genFileStructure();
+    outputFiles(new File(outDir, FILE_STRUCTURE_FILE_NAME));
+    return exitCode;
+  }
+
+  /** Parse the command line arguments and initialize the data */
+  private int init(String[] args) {
+    try {
+      for (int i = 0; i < args.length; i++) { // parse command line
+        if (args[i].equals("-maxDepth")) {
+          maxDepth = Integer.parseInt(args[++i]);
+          if (maxDepth<1) {
+            System.err.println("maxDepth must be positive: " + maxDepth);
+            return -1;
+          }
+        } else if (args[i].equals("-minWidth")) {
+          minWidth = Integer.parseInt(args[++i]);
+          if (minWidth<0) {
+            System.err.println("minWidth must be positive: " + minWidth);
+            return -1;
+          }
+        } else if (args[i].equals("-maxWidth")) {
+          maxWidth = Integer.parseInt(args[++i]);
+        } else if (args[i].equals("-numOfFiles")) {
+          numOfFiles = Integer.parseInt(args[++i]);
+          if (numOfFiles<1) {
+            System.err.println("NumOfFiles must be positive: " + numOfFiles);
+            return -1;
+          }
+        } else if (args[i].equals("-avgFileSize")) {
+          avgFileSize = Double.parseDouble(args[++i]);
+          if (avgFileSize<=0) {
+            System.err.println("AvgFileSize must be positive: " + avgFileSize);
+            return -1;
+          }
+        } else if (args[i].equals("-outDir")) {
+          outDir = new File(args[++i]);
+        } else if (args[i].equals("-seed")) {
+          r = new Random(Long.parseLong(args[++i]));
+        } else {
+          System.err.println(USAGE);
+          ToolRunner.printGenericCommandUsage(System.err);
+          return -1;
+        }
+      }
+    } catch (NumberFormatException e) {
+      System.err.println("Illegal parameter: " + e.getLocalizedMessage());
+      System.err.println(USAGE);
+      return -1;
+    }
+    
+    if (maxWidth < minWidth) {
+      System.err.println(
+          "maxWidth must be bigger than minWidth: " + maxWidth);
+      return -1;
+    }
+    
+    if (r==null) {
+      r = new Random();
+    }
+    return 0;
+  }
+  
+  /** In memory representation of a directory */
+  private static class INode {
+    private String name;
+    private List<INode> children = new ArrayList<INode>();
+    
+    /** Constructor */
+    private INode(String name) {
+      this.name = name;
+    }
+    
+    /** Add a child (subdir/file) */
+    private void addChild(INode child) {
+      children.add(child);
+    }
+    
+    /** Output the subtree rooted at the current node. 
+     * Only the leaves are printed.
+     */
+    private void output(PrintStream out, String prefix) {
+      prefix = prefix==null?name:prefix+"/"+name;
+      if (children.isEmpty()) {
+        out.println(prefix);
+      } else {
+        for (INode child : children) {
+          child.output(out, prefix);
+        }
+      }
+    }
+    
+    /** Output the files in the subtree rooted at this node */
+    protected void outputFiles(PrintStream out, String prefix) {
+      prefix = prefix==null?name:prefix+"/"+name;
+      for (INode child : children) {
+        child.outputFiles(out, prefix);
+      }
+    }
+    
+    /** Add all the leaves in the subtree to the input list */
+    private void getLeaves(List<INode> leaves) {
+      if (children.isEmpty()) {
+        leaves.add(this);
+      } else {
+        for (INode child : children) {
+          child.getLeaves(leaves);
+        }
+      }
+    }
+  }
+  
+  /** In memory representation of a file */
+  private static class FileINode extends INode {
+    private double numOfBlocks;
+
+    /** constructor */
+    private FileINode(String name, double numOfBlocks) {
+      super(name);
+      this.numOfBlocks = numOfBlocks;
+    }
+    
+    /** Output a file attribute */
+    protected void outputFiles(PrintStream out, String prefix) {
+      prefix = (prefix == null)?super.name: prefix + "/"+super.name;
+      out.println(prefix + " " + numOfBlocks);
+    }
+  }
+
+  private INode root;
+  
+  /** Generates a directory tree with a max depth of <code>maxDepth</code> */
+  private void genDirStructure() {
+    root = genDirStructure("", maxDepth);
+  }
+  
+  /** Generate a directory tree rooted at <code>rootName</code>
+   * The number of subtree is in the range of [minWidth, maxWidth].
+   * The maximum depth of each subtree is in the range of
+   * [2*maxDepth/3, maxDepth].
+   */
+  private INode genDirStructure(String rootName, int maxDepth) {
+    INode root = new INode(rootName);
+    
+    if (maxDepth>0) {
+      maxDepth--;
+      int minDepth = maxDepth*2/3;
+      // Figure out the number of subdirectories to generate
+      int numOfSubDirs = minWidth + r.nextInt(maxWidth-minWidth+1);
+      // Expand the tree
+      for (int i=0; i<numOfSubDirs; i++) {
+        int childDepth = (maxDepth == 0)?0:
+          (r.nextInt(maxDepth-minDepth+1)+minDepth);
+        INode child = genDirStructure("dir"+i, childDepth);
+        root.addChild(child);
+      }
+    }
+    return root;
+  }
+  
+  /** Collects leaf nodes in the tree */
+  private List<INode> getLeaves() {
+    List<INode> leaveDirs = new ArrayList<INode>();
+    root.getLeaves(leaveDirs);
+    return leaveDirs;
+  }
+  
+  /** Decides where to place all the files and its length.
+   * It first collects all empty directories in the tree.
+   * For each file, it randomly chooses an empty directory to place the file.
+   * The file's length is generated using Gaussian distribution.
+   */
+  private void genFileStructure() {
+    List<INode> leaves = getLeaves();
+    int totalLeaves = leaves.size();
+    for (int i=0; i<numOfFiles; i++) {
+      int leaveNum = r.nextInt(totalLeaves);
+      double fileSize;
+      do {
+        fileSize = r.nextGaussian()+avgFileSize;
+      } while (fileSize<0);
+      leaves.get(leaveNum).addChild(
+          new FileINode(FILE_NAME_PREFIX+i, fileSize));
+    }
+  }
+  
+  /** Output directory structure to a file, each line of the file
+   * contains the directory name. Only empty directory names are printed. */
+  private void output(File outFile) throws FileNotFoundException {
+    System.out.println("Printing to " + outFile.toString());
+    PrintStream out = new PrintStream(outFile);
+    root.output(out, null);
+    out.close();
+  }
+  
+  /** Output all files' attributes to a file, each line of the output file
+   * contains a file name and its length. */
+  private void outputFiles(File outFile) throws FileNotFoundException {
+    System.out.println("Printing to " + outFile.toString());
+    PrintStream out = new PrintStream(outFile);
+    root.outputFiles(out, null);
+    out.close();
+  }
+  
+  /**
+   * Main program
+   * @param args Command line arguments
+   * @throws Exception
+   */
+  public static void main(String[] args) throws Exception {
+    StructureGenerator sg = new StructureGenerator();
+    System.exit(sg.run(args));
+  }
+}

Added: hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/TestLoadGenerator.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/TestLoadGenerator.java?rev=694458&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/TestLoadGenerator.java (added)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/TestLoadGenerator.java Thu
Sep 11 13:23:09 2008
@@ -0,0 +1,217 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.loadGenerator;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+
+import junit.framework.TestCase;
+/**
+ * This class tests if a balancer schedules tasks correctly.
+ */
+public class TestLoadGenerator extends TestCase {
+  private static final Configuration CONF = new Configuration();
+  private static final int DEFAULT_BLOCK_SIZE = 10;
+  private static final String OUT_DIR = 
+    System.getProperty("test.build.data","build/test/data");
+  private static final File DIR_STRUCTURE_FILE = 
+    new File(OUT_DIR, StructureGenerator.DIR_STRUCTURE_FILE_NAME);
+  private static final File FILE_STRUCTURE_FILE =
+    new File(OUT_DIR, StructureGenerator.FILE_STRUCTURE_FILE_NAME);
+  private static final String DIR_STRUCTURE_FIRST_LINE = "/dir0";
+  private static final String DIR_STRUCTURE_SECOND_LINE = "/dir1";
+  private static final String FILE_STRUCTURE_FIRST_LINE =
+    "/dir0/_file_0 0.3754598635933768";
+  private static final String FILE_STRUCTURE_SECOND_LINE =
+    "/dir1/_file_1 1.4729310851145203";
+  
+
+  static {
+    CONF.setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
+    CONF.setInt("io.bytes.per.checksum", DEFAULT_BLOCK_SIZE);
+    CONF.setLong("dfs.heartbeat.interval", 1L);
+  }
+
+  /** Test if the structure generator works fine */ 
+  public void testStructureGenerator() throws Exception {
+    StructureGenerator sg = new StructureGenerator();
+    String[] args = new String[]{"-maxDepth", "2", "-minWidth", "1",
+        "-maxWidth", "2", "-numOfFiles", "2",
+        "-avgFileSize", "1", "-outDir", OUT_DIR, "-seed", "1"};
+    
+    final int MAX_DEPTH = 1;
+    final int MIN_WIDTH = 3;
+    final int MAX_WIDTH = 5;
+    final int NUM_OF_FILES = 7;
+    final int AVG_FILE_SIZE = 9;
+    final int SEED = 13;
+    try {
+      // successful case
+      assertEquals(0, sg.run(args));
+      BufferedReader in = new BufferedReader(new FileReader(DIR_STRUCTURE_FILE));
+      assertEquals(DIR_STRUCTURE_FIRST_LINE, in.readLine());
+      assertEquals(DIR_STRUCTURE_SECOND_LINE, in.readLine());
+      assertEquals(null, in.readLine());
+      in.close();
+      
+      in = new BufferedReader(new FileReader(FILE_STRUCTURE_FILE));
+      assertEquals(FILE_STRUCTURE_FIRST_LINE, in.readLine());
+      assertEquals(FILE_STRUCTURE_SECOND_LINE, in.readLine());
+      assertEquals(null, in.readLine());
+      in.close();
+
+      String oldArg = args[MAX_DEPTH];
+      args[MAX_DEPTH] = "0";
+      assertEquals(-1, sg.run(args));
+      args[MAX_DEPTH] = oldArg;
+      
+      oldArg = args[MIN_WIDTH];
+      args[MIN_WIDTH] = "-1";
+      assertEquals(-1, sg.run(args));
+      args[MIN_WIDTH] = oldArg;
+      
+      oldArg = args[MAX_WIDTH];
+      args[MAX_WIDTH] = "-1";
+      assertEquals(-1, sg.run(args));
+      args[MAX_WIDTH] = oldArg;
+      
+      oldArg = args[NUM_OF_FILES];
+      args[NUM_OF_FILES] = "-1";
+      assertEquals(-1, sg.run(args));
+      args[NUM_OF_FILES] = oldArg;
+      
+      oldArg = args[NUM_OF_FILES];
+      args[NUM_OF_FILES] = "-1";
+      assertEquals(-1, sg.run(args));
+      args[NUM_OF_FILES] = oldArg;
+      
+      oldArg = args[AVG_FILE_SIZE];
+      args[AVG_FILE_SIZE] = "-1";
+      assertEquals(-1, sg.run(args));
+      args[AVG_FILE_SIZE] = oldArg;
+      
+      oldArg = args[SEED];
+      args[SEED] = "34.d4";
+      assertEquals(-1, sg.run(args));
+      args[SEED] = oldArg;
+    } finally {
+      DIR_STRUCTURE_FILE.delete();
+      FILE_STRUCTURE_FILE.delete();
+    }
+  }
+
+  /** Test if the load generator works fine */
+  public void testLoadGenerator() throws Exception {
+    final String TEST_SPACE_ROOT = "/test";
+
+    FileWriter writer = new FileWriter(DIR_STRUCTURE_FILE);
+    writer.write(DIR_STRUCTURE_FIRST_LINE+"\n");
+    writer.write(DIR_STRUCTURE_SECOND_LINE+"\n");
+    writer.close();
+    
+    writer = new FileWriter(FILE_STRUCTURE_FILE);
+    writer.write(FILE_STRUCTURE_FIRST_LINE+"\n");
+    writer.write(FILE_STRUCTURE_SECOND_LINE+"\n");
+    writer.close();
+    
+    MiniDFSCluster cluster = new MiniDFSCluster(CONF, 3, true, null);
+    cluster.waitActive();
+    
+    try {
+      DataGenerator dg = new DataGenerator();
+      dg.setConf(CONF);
+      String [] args = new String[] {"-inDir", OUT_DIR, "-root", TEST_SPACE_ROOT};
+      assertEquals(0, dg.run(args));
+
+      final int READ_PROBABILITY = 1;
+      final int WRITE_PROBABILITY = 3;
+      final int MAX_DELAY_BETWEEN_OPS = 7;
+      final int NUM_OF_THREADS = 9;
+      final int START_TIME = 11;
+      final int ELAPSED_TIME = 13;
+      
+      LoadGenerator lg = new LoadGenerator();
+      lg.setConf(CONF);
+      args = new String[] {"-readProbability", "0.3", "-writeProbability", "0.3",
+          "-root", TEST_SPACE_ROOT, "-maxDelayBetweenOps", "0",
+          "-numOfThreads", "1", "-startTime", 
+          Long.toString(System.currentTimeMillis()), "-elapsedTime", "10"};
+      
+      assertEquals(0, lg.run(args));
+
+      String oldArg = args[READ_PROBABILITY];
+      args[READ_PROBABILITY] = "1.1";
+      assertEquals(-1, lg.run(args));
+      args[READ_PROBABILITY] = "-1.1";
+      assertEquals(-1, lg.run(args));
+      args[READ_PROBABILITY] = oldArg;
+
+      oldArg = args[WRITE_PROBABILITY];
+      args[WRITE_PROBABILITY] = "1.1";
+      assertEquals(-1, lg.run(args));
+      args[WRITE_PROBABILITY] = "-1.1";
+      assertEquals(-1, lg.run(args));
+      args[WRITE_PROBABILITY] = "0.9";
+      assertEquals(-1, lg.run(args));
+      args[READ_PROBABILITY] = oldArg;
+
+      oldArg = args[MAX_DELAY_BETWEEN_OPS];
+      args[MAX_DELAY_BETWEEN_OPS] = "1.x1";
+      assertEquals(-1, lg.run(args));
+      args[MAX_DELAY_BETWEEN_OPS] = oldArg;
+      
+      oldArg = args[MAX_DELAY_BETWEEN_OPS];
+      args[MAX_DELAY_BETWEEN_OPS] = "1.x1";
+      assertEquals(-1, lg.run(args));
+      args[MAX_DELAY_BETWEEN_OPS] = oldArg;
+      
+      oldArg = args[NUM_OF_THREADS];
+      args[NUM_OF_THREADS] = "-1";
+      assertEquals(-1, lg.run(args));
+      args[NUM_OF_THREADS] = oldArg;
+      
+      oldArg = args[START_TIME];
+      args[START_TIME] = "-1";
+      assertEquals(-1, lg.run(args));
+      args[START_TIME] = oldArg;
+
+      oldArg = args[ELAPSED_TIME];
+      args[ELAPSED_TIME] = "-1";
+      assertEquals(-1, lg.run(args));
+      args[ELAPSED_TIME] = oldArg;
+    } finally {
+      cluster.shutdown();
+      DIR_STRUCTURE_FILE.delete();
+      FILE_STRUCTURE_FILE.delete();
+    }
+  }
+  
+  /**
+   * @param args
+   */
+  public static void main(String[] args) throws Exception {
+    TestLoadGenerator loadGeneratorTest = new TestLoadGenerator();
+    loadGeneratorTest.testStructureGenerator();
+    loadGeneratorTest.testLoadGenerator();
+  }
+}



Mime
View raw message