hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maha...@apache.org
Subject svn commit: r678845 - in /hadoop/core/trunk: CHANGES.txt src/test/org/apache/hadoop/hdfs/CreateEditsLog.java src/test/org/apache/hadoop/hdfs/FileNameGenerator.java src/test/org/apache/hadoop/hdfs/NNThroughputBenchmark.java
Date Tue, 22 Jul 2008 18:23:29 GMT
Author: mahadev
Date: Tue Jul 22 11:23:28 2008
New Revision: 678845

URL: http://svn.apache.org/viewvc?rev=678845&view=rev
Log:
HADOOP-3624. Improving createeditslog to create tree directory structure. (Lohit Vijayarenu
via mahadev)

Added:
    hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/FileNameGenerator.java
Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/CreateEditsLog.java
    hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/NNThroughputBenchmark.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=678845&r1=678844&r2=678845&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue Jul 22 11:23:28 2008
@@ -155,6 +155,9 @@
 
     HADOOP-3795. Fix saving image files on Namenode with different checkpoint
     stamps. (Lohit Vijayarenu via mahadev)
+   
+    HADOOP-3624. Improving createeditslog to create tree directory structure.
+    (Lohit Vijayarenu via mahadev)
 
 Release 0.18.0 - Unreleased
 

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/CreateEditsLog.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/CreateEditsLog.java?rev=678845&r1=678844&r2=678845&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/CreateEditsLog.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/CreateEditsLog.java Tue Jul 22 11:23:28
2008
@@ -56,14 +56,12 @@
 
 public class CreateEditsLog {
   static final String BASE_PATH = "/createdViaInjectingInEditsLog";
-  static final String BASE_PATH_PLUS_FILEPREFIX = BASE_PATH + "/file";
-  
   static final String EDITS_DIR = "/tmp/EditsLogOut";
-
   static String edits_dir = EDITS_DIR;
   
   static void addFiles(FSEditLog editLog, int numFiles, short replication, 
-                                  int blocksPerFile, long startingBlockId) {
+                         int blocksPerFile, long startingBlockId,
+                         FileNameGenerator nameGenerator) {
     
     PermissionStatus p = new PermissionStatus("joeDoe", "people",
                                       new FsPermission((short)0777));
@@ -80,7 +78,7 @@
     long currentBlockId = startingBlockId;
     long bidAtSync = startingBlockId;
 
-    for (int iF = 1; iF <= numFiles; iF++) {
+    for (int iF = 0; iF < numFiles; iF++) {
       for (int iB = 0; iB < blocksPerFile; ++iB) {
          blocks[iB].setBlockId(currentBlockId++);
       }
@@ -89,11 +87,16 @@
 
         INodeFileUnderConstruction inode = new INodeFileUnderConstruction(
                       null, replication, 0, blockSize, blocks, p, "", "", null);
-        String path = BASE_PATH_PLUS_FILEPREFIX  + iF +
-          "_B" + blocks[0].getBlockId() + 
-          "_to_B" + blocks[blocksPerFile-1].getBlockId();
-        editLog.logOpenFile(path, inode);
-        editLog.logCloseFile(path, inode);
+        String path = iF + "_B" + blocks[0].getBlockId() + 
+                      "_to_B" + blocks[blocksPerFile-1].getBlockId() + "_";
+        String filePath = nameGenerator.getNextFileName(path);
+        if ((iF % nameGenerator.getFilesPerDirectory())  == 0) {
+          String currentDir = nameGenerator.getCurrentDir();
+          dirInode = new INodeDirectory(p, 0L);
+          editLog.logMkDir(currentDir, dirInode);
+        }
+        editLog.logOpenFile(filePath, inode);
+        editLog.logCloseFile(filePath, inode);
 
         if (currentBlockId - bidAtSync >= 2000) { // sync every 2K blocks
           editLog.logSync();
@@ -145,6 +148,8 @@
     }
 
     for (int i = 0; i < args.length; i++) { // parse command line
+      if (args[i].equals("-h"))
+        printUsageExit();
       if (args[i].equals("-f")) {
        if (i + 3 >= args.length || args[i+1].startsWith("-") || 
            args[i+2].startsWith("-") || args[i+3].startsWith("-")) {
@@ -191,12 +196,14 @@
     }
   
     FSImage fsImage = new FSImage(new File(edits_dir));
+    FileNameGenerator nameGenerator = new FileNameGenerator(BASE_PATH, 100);
 
 
     FSEditLog editLog = fsImage.getEditLog();
     editLog.createEditLogFile(fsImage.getFsEditName());
     editLog.open();
-    addFiles(editLog, numFiles, replication, numBlocksPerFile, startingBlockId);
+    addFiles(editLog, numFiles, replication, numBlocksPerFile, startingBlockId,
+             nameGenerator);
     editLog.logSync();
     editLog.close();
   }

Added: hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/FileNameGenerator.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/FileNameGenerator.java?rev=678845&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/FileNameGenerator.java (added)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/FileNameGenerator.java Tue Jul 22 11:23:28
2008
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import java.util.Arrays;
+
+/**
+ * File name generator.
+ * 
+ * Each directory contains not more than a fixed number (filesPerDir) 
+ * of files and directories.
+ * When the number of files in one directory reaches the maximum,
+ * the generator creates a new directory and proceeds generating files in it.
+ * The generated namespace tree is balanced that is any path to a leaf
+ * file is not less than the height of the tree minus one.
+ */
+public class FileNameGenerator {
+  private static final int DEFAULT_FILES_PER_DIRECTORY = 32;
+  
+  private int[] pathIndecies = new int[20]; // this will support up to 32**20 = 2**100 =
10**30 files
+  private String baseDir;
+  private String currentDir;
+  private int filesPerDirectory;
+  private long fileCount;
+
+  FileNameGenerator(String baseDir) {
+    this(baseDir, DEFAULT_FILES_PER_DIRECTORY);
+  }
+  
+  FileNameGenerator(String baseDir, int filesPerDir) {
+    this.baseDir = baseDir;
+    this.filesPerDirectory = filesPerDir;
+    reset();
+  }
+
+  String getNextDirName(String prefix) {
+    int depth = 0;
+    while(pathIndecies[depth] >= 0)
+      depth++;
+    int level;
+    for(level = depth-1; 
+        level >= 0 && pathIndecies[level] == filesPerDirectory-1; level--)
+      pathIndecies[level] = 0;
+    if(level < 0)
+      pathIndecies[depth] = 0;
+    else
+      pathIndecies[level]++;
+    level = 0;
+    String next = baseDir;
+    while(pathIndecies[level] >= 0)
+      next = next + "/" + prefix + pathIndecies[level++];
+    return next; 
+  }
+
+  synchronized String getNextFileName(String fileNamePrefix) {
+    long fNum = fileCount % filesPerDirectory;
+    if(fNum == 0) {
+      currentDir = getNextDirName(fileNamePrefix + "Dir");
+    }
+    String fn = currentDir + "/" + fileNamePrefix + fileCount;
+    fileCount++;
+    return fn;
+  }
+
+  private synchronized void reset() {
+    Arrays.fill(pathIndecies, -1);
+    fileCount = 0L;
+    currentDir = "";
+  }
+
+  synchronized int getFilesPerDirectory() {
+    return filesPerDirectory;
+  }
+
+  synchronized String getCurrentDir() {
+    return currentDir;
+  }
+}

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/NNThroughputBenchmark.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/NNThroughputBenchmark.java?rev=678845&r1=678844&r2=678845&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/NNThroughputBenchmark.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/NNThroughputBenchmark.java Tue Jul 22
11:23:28 2008
@@ -365,71 +365,6 @@
   }
 
   /**
-   * File name generator.
-   * 
-   * Each directory contains not more than a fixed number (filesPerDir) 
-   * of files and directories.
-   * When the number of files in one directory reaches the maximum,
-   * the generator creates a new directory and proceeds generating files in it.
-   * The generated namespace tree is balanced that is any path to a leaf
-   * file is not less than the height of the tree minus one.
-   */
-  private static class FileGenerator {
-    private static final int DEFAULT_FILES_PER_DIRECTORY = 32;
-    // Average file name size is 16.5 bytes
-    private static final String FILE_NAME_PREFFIX ="ThrouputBenchfile"; // 17 bytes
-    private static final String DIR_NAME_PREFFIX = "ThrouputBenchDir";  // 16 bytes
-    // private static final int NUM_CLIENTS = 100;
-
-    private int[] pathIndecies = new int[20]; // this will support up to 32**20 = 2**100
= 10**30 files
-    private String baseDir;
-    private String currentDir;
-    private int filesPerDirectory = DEFAULT_FILES_PER_DIRECTORY;
-    private long fileCount;
-
-    FileGenerator(String baseDir, int filesPerDir) {
-      this.baseDir = baseDir;
-      this.filesPerDirectory = filesPerDir;
-      reset();
-    }
-
-    String getNextDirName() {
-      int depth = 0;
-      while(pathIndecies[depth] >= 0)
-        depth++;
-      int level;
-      for(level = depth-1; 
-          level >= 0 && pathIndecies[level] == filesPerDirectory-1; level--)
-        pathIndecies[level] = 0;
-      if(level < 0)
-        pathIndecies[depth] = 0;
-      else
-        pathIndecies[level]++;
-      level = 0;
-      String next = baseDir;
-      while(pathIndecies[level] >= 0)
-        next = next + "/" + DIR_NAME_PREFFIX + pathIndecies[level++];
-      return next; 
-    }
-
-    synchronized String getNextFileName() {
-      long fNum = fileCount % filesPerDirectory;
-      if(fNum == 0) {
-        currentDir = getNextDirName();
-      }
-      String fn = currentDir + "/" + FILE_NAME_PREFFIX + fileCount;
-      fileCount++;
-      return fn;
-    }
-
-    private synchronized void reset() {
-      Arrays.fill(pathIndecies, -1);
-      fileCount = 0L;
-      currentDir = "";
-    }
-  }
-
-  /**
    * File creation statistics.
    * 
    * Each thread creates the same (+ or -1) number of files.
@@ -442,7 +377,7 @@
     static final String OP_CREATE_USAGE = 
       "-op create [-threads T] [-files N] [-filesPerDir P]";
 
-    protected FileGenerator nameGenerator;
+    protected FileNameGenerator nameGenerator;
     protected String[][] fileNames;
 
     CreateFileStats(String[] args) {
@@ -470,7 +405,7 @@
         } else if(!ignoreUnrelatedOptions)
           printUsage();
       }
-      nameGenerator = new FileGenerator(getBaseDir(), nrFilesPerDir);
+      nameGenerator = new FileNameGenerator(getBaseDir(), nrFilesPerDir);
     }
 
     void generateInputs(int[] opsPerThread) throws IOException {
@@ -483,7 +418,8 @@
         int threadOps = opsPerThread[idx];
         fileNames[idx] = new String[threadOps];
         for(int jdx=0; jdx < threadOps; jdx++)
-          fileNames[idx][jdx] = nameGenerator.getNextFileName();
+          fileNames[idx][jdx] = nameGenerator.
+                                  getNextFileName("ThroughputBench");
       }
     }
 
@@ -516,7 +452,7 @@
       LOG.info("--- " + getOpName() + " inputs ---");
       LOG.info("nrFiles = " + numOpsRequired);
       LOG.info("nrThreads = " + numThreads);
-      LOG.info("nrFilesPerDir = " + nameGenerator.filesPerDirectory);
+      LOG.info("nrFilesPerDir = " + nameGenerator.getFilesPerDirectory());
       printStats();
     }
   }
@@ -548,7 +484,8 @@
               "-op", "create", 
               "-threads", String.valueOf(this.numThreads), 
               "-files", String.valueOf(numOpsRequired),
-              "-filesPerDir", String.valueOf(nameGenerator.filesPerDirectory)};
+              "-filesPerDir", 
+              String.valueOf(nameGenerator.getFilesPerDirectory())};
       CreateFileStats opCreate =  new CreateFileStats(createArgs);
       opCreate.benchmark();
       LOG.info("Created " + numOpsRequired + " files.");
@@ -782,12 +719,12 @@
 
       // create files 
       LOG.info("Creating " + nrFiles + " with " + blocksPerFile + " blocks each.");
-      FileGenerator nameGenerator;
-      nameGenerator = new FileGenerator(getBaseDir(), 100);
+      FileNameGenerator nameGenerator;
+      nameGenerator = new FileNameGenerator(getBaseDir(), 100);
       String clientName = getClientName(007);
       nameNode.setSafeMode(FSConstants.SafeModeAction.SAFEMODE_LEAVE);
       for(int idx=0; idx < nrFiles; idx++) {
-        String fileName = nameGenerator.getNextFileName();
+        String fileName = nameGenerator.getNextFileName("ThroughputBench");
         nameNode.create(fileName, FsPermission.getDefault(),
                         clientName, true, replication, BLOCK_SIZE);
         addBlocks(fileName, clientName);



Mime
View raw message