hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From whe...@apache.org
Subject svn commit: r1573696 - in /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/
Date Mon, 03 Mar 2014 19:42:22 GMT
Author: wheat9
Date: Mon Mar  3 19:42:22 2014
New Revision: 1573696

URL: http://svn.apache.org/r1573696
Log:
Merge r1573694 from trunk.

Modified:
    hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
    hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java
    hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
    hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java

Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1573696&r1=1573695&r2=1573696&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Mar  3
19:42:22 2014
@@ -270,6 +270,9 @@ Release 2.4.0 - UNRELEASED
     HDFS-5956. A file size is multiplied by the replication factor in 'hdfs oiv
     -p FileDistribution' option. (Akira Ajisaka via wheat9)
 
+    HDFS-5866. '-maxSize' and '-step' option fail in OfflineImageViewer.
+    (Akira Ajisaka via wheat9)
+
   BREAKDOWN OF HDFS-5698 SUBTASKS AND RELATED JIRAS
 
     HDFS-5717. Save FSImage header in protobuf. (Haohui Mai via jing9)

Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java?rev=1573696&r1=1573695&r2=1573696&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java
Mon Mar  3 19:42:22 2014
@@ -62,6 +62,7 @@ import com.google.common.io.LimitInputSt
 final class FileDistributionCalculator {
   private final static long MAX_SIZE_DEFAULT = 0x2000000000L; // 1/8 TB = 2^37
   private final static int INTERVAL_DEFAULT = 0x200000; // 2 MB = 2^21
+  private final static int MAX_INTERVALS = 0x8000000; // 128 M = 2^27
 
   private final Configuration conf;
   private final long maxSize;
@@ -82,9 +83,11 @@ final class FileDistributionCalculator {
     this.steps = steps == 0 ? INTERVAL_DEFAULT : steps;
     this.out = out;
     long numIntervals = this.maxSize / this.steps;
+    // avoid OutOfMemoryError when allocating an array
+    Preconditions.checkState(numIntervals <= MAX_INTERVALS,
+        "Too many distribution intervals (maxSize/step): " + numIntervals +
+        ", should be less than " + (MAX_INTERVALS+1) + ".");
     this.distribution = new int[1 + (int) (numIntervals)];
-    Preconditions.checkState(numIntervals < Integer.MAX_VALUE,
-        "Too many distribution intervals");
   }
 
   void visit(RandomAccessFile file) throws IOException {

Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java?rev=1573696&r1=1573695&r2=1573696&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
Mon Mar  3 19:42:22 2014
@@ -101,9 +101,8 @@ public class OfflineImageViewerPB {
 
     options.addOption("p", "processor", true, "");
     options.addOption("h", "help", false, "");
-    options.addOption("skipBlocks", false, "");
-    options.addOption("printToScreen", false, "");
-    options.addOption("delimiter", true, "");
+    options.addOption("maxSize", true, "");
+    options.addOption("step", true, "");
 
     return options;
   }
@@ -118,10 +117,15 @@ public class OfflineImageViewerPB {
    * @throws IOException
    */
   public static void main(String[] args) throws IOException {
+    int status = run(args);
+    System.exit(status);
+  }
+
+  public static int run(String[] args) throws IOException {
     Options options = buildOptions();
     if (args.length == 0) {
       printUsage();
-      return;
+      return 0;
     }
 
     CommandLineParser parser = new PosixParser();
@@ -132,12 +136,12 @@ public class OfflineImageViewerPB {
     } catch (ParseException e) {
       System.out.println("Error parsing command-line options: ");
       printUsage();
-      return;
+      return -1;
     }
 
     if (cmd.hasOption("h")) { // print help and exit
       printUsage();
-      return;
+      return 0;
     }
 
     String inputFile = cmd.getOptionValue("i");
@@ -160,6 +164,7 @@ public class OfflineImageViewerPB {
       } else {
         new LsrPBImage(conf, out).visit(new RandomAccessFile(inputFile, "r"));
       }
+      return 0;
     } catch (EOFException e) {
       System.err.println("Input file ended unexpectedly. Exiting");
     } catch (IOException e) {
@@ -167,7 +172,7 @@ public class OfflineImageViewerPB {
     } finally {
       IOUtils.cleanup(null, out);
     }
-
+    return -1;
   }
 
   /**

Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java?rev=1573696&r1=1573695&r2=1573696&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
Mon Mar  3 19:42:22 2014
@@ -278,6 +278,14 @@ public class TestOfflineImageViewer {
   }
 
   @Test
+  public void testFileDistributionCalculatorWithOptions() throws IOException {
+    int status = OfflineImageViewerPB.run(new String[] {"-i",
+        originalFsimage.getAbsolutePath(), "-o", "-", "-p", "FileDistribution",
+        "-maxSize", "512", "-step", "8"});
+    assertEquals(0, status);
+  }
+
+  @Test
   public void testPBImageXmlWriter() throws IOException, SAXException,
       ParserConfigurationException {
     StringWriter output = new StringWriter();



Mime
View raw message