hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From c..@apache.org
Subject svn commit: r1041261 - in /hadoop/hdfs/trunk: CHANGES.txt src/test/hdfs/org/apache/hadoop/hdfs/TestLargeBlock.java
Date Thu, 02 Dec 2010 02:52:04 GMT
Author: cos
Date: Thu Dec  2 02:52:04 2010
New Revision: 1041261

URL: http://svn.apache.org/viewvc?rev=1041261&view=rev
Log:
HDFS-1534. TestLargeBlock is failing on trunk. Contributed by Konstantin Boudnik

Modified:
    hadoop/hdfs/trunk/CHANGES.txt
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestLargeBlock.java

Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1041261&r1=1041260&r2=1041261&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Thu Dec  2 02:52:04 2010
@@ -422,6 +422,8 @@ Release 0.22.0 - Unreleased
     HDFS-1524. Image loader should make sure to read every byte in image file.
     (hairong)
 
+    HDFS-1534. TestLargeBlock is failing on trunk. (cos)
+
 Release 0.21.1 - Unreleased
 
   IMPROVEMENTS

Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestLargeBlock.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestLargeBlock.java?rev=1041261&r1=1041260&r2=1041261&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestLargeBlock.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestLargeBlock.java Thu Dec  2
02:52:04 2010
@@ -17,72 +17,70 @@
  */
 package org.apache.hadoop.hdfs;
 
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
 import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.Random;
 import java.util.Arrays;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.log4j.Level;
+
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.FSConstants;
-import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.datanode.FSDataset;
 import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.log4j.Level;
 
 import org.junit.Test;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
 
 /**
  * This class tests that blocks can be larger than 2GB
  */
-public class TestLargeBlock extends junit.framework.TestCase {
-  static final String DIR = "/" + TestLargeBlock.class.getSimpleName() + "/";
-
+public class TestLargeBlock {
+/**
   {
-    // ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL);
-    // ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL);
-    // ((Log4JLogger)FSNamesystem.LOG).getLogger().setLevel(Level.ALL);
-    // ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL);
+    ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL);
+    ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL);
+    ((Log4JLogger)FSNamesystem.LOG).getLogger().setLevel(Level.ALL);
+    ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL);
+    ((Log4JLogger)TestLargeBlock.LOG).getLogger().setLevel(Level.ALL);
   }
+ */
+  private static final Log LOG = LogFactory.getLog(TestLargeBlock.class);
 
-
-  static final boolean verifyData = true; // should we verify the data read back from the
file? (slow)
+  // should we verify the data read back from the file? (slow)
+  static final boolean verifyData = true;
   static final byte[] pattern = { 'D', 'E', 'A', 'D', 'B', 'E', 'E', 'F'};
   static final boolean simulatedStorage = false;
+  private static final String ALLOWED_VM = "64";
 
   // creates a file 
-  static FSDataOutputStream createFile(FileSystem fileSys, Path name, int repl, final long
blockSize)
+  static FSDataOutputStream createFile(FileSystem fileSys, Path name, int repl,
+                                       final long blockSize)
     throws IOException {
     FSDataOutputStream stm = fileSys.create(name, true,
-                                            fileSys.getConf().getInt("io.file.buffer.size",
4096),
-                                            (short)repl, blockSize);
-    System.out.println("createFile: Created " + name + " with " + repl + " replica.");
+        fileSys.getConf().getInt("io.file.buffer.size", 4096),
+        (short)repl, blockSize);
+    LOG.info("createFile: Created " + name + " with " + repl + " replica.");
     return stm;
   }
 
-
   /**
    * Writes pattern to file
+   * @param stm FSDataOutputStream to write the file
+   * @param fileSize size of the file to be written
+   * @throws IOException in case of errors
    */
-  static void writeFile(FSDataOutputStream stm, final long fileSize) throws IOException {
-    final int writeSize = pattern.length * 8 * 1024 * 1024; // write in chunks of 64 MB
-    final int writeCount = (int) ((fileSize / ((long) writeSize)) + ((fileSize % ((long)
writeSize) == 0L) ? 0L : 1L));
+  static void writeFile(FSDataOutputStream stm, final long fileSize)
+      throws IOException {
+    // write in chunks of 64 MB
+    final int writeSize = pattern.length * 8 * 1024 * 1024;
 
     if (writeSize > Integer.MAX_VALUE) {
       throw new IOException("A single write is too large " + writeSize);
@@ -96,24 +94,26 @@ public class TestLargeBlock extends juni
       b[j] = pattern[j % pattern.length];
     }
 
-    int i = 0;
-
     while (bytesToWrite > 0) {
-      int thiswrite = (int) Math.min(writeSize, bytesToWrite); // how many bytes we are writing
in this iteration
+      // how many bytes we are writing in this iteration
+      int thiswrite = (int) Math.min(writeSize, bytesToWrite);
 
       stm.write(b, 0, thiswrite);
-      // System.out.println("Wrote[" + i + "/" + writeCount + "] " + thiswrite + " bytes.");
       bytesToWrite -= thiswrite;
-      i++;
     }
   }
 
   /**
    * Reads from file and makes sure that it matches the pattern
+   * @param fs a reference to FileSystem
+   * @param name Path of a file
+   * @param fileSize size of the file
+   * @throws IOException in case of errors
    */
-  static void checkFullFile(FileSystem fs, Path name, final long fileSize) throws IOException
{
-    final int readSize = pattern.length * 16 * 1024 * 1024; // read in chunks of 128 MB
-    final int readCount = (int) ((fileSize / ((long) readSize)) + ((fileSize % ((long) readSize)
== 0L) ? 0L : 1L));
+  static void checkFullFile(FileSystem fs, Path name, final long fileSize)
+      throws IOException {
+    // read in chunks of 128 MB
+    final int readSize = pattern.length * 16 * 1024 * 1024;
 
     if (readSize > Integer.MAX_VALUE) {
       throw new IOException("A single read is too large " + readSize);
@@ -131,48 +131,51 @@ public class TestLargeBlock extends juni
       }
     }
 
-
     FSDataInputStream stm = fs.open(name);
 
-    int i = 0;
-
     while (bytesToRead > 0) {
-      int thisread = (int) Math.min(readSize, bytesToRead); // how many bytes we are reading
in this iteration
+      // how many bytes we are reading in this iteration
+      int thisread = (int) Math.min(readSize, bytesToRead);
 
       stm.readFully(b, 0, thisread); 
       
       if (verifyData) {
         // verify data read
-        
         if (thisread == readSize) {
-          assertTrue("file corrupted at or after byte " + (fileSize - bytesToRead), Arrays.equals(b,
compb));
+          assertTrue("file is corrupted at or after byte " +
+              (fileSize - bytesToRead), Arrays.equals(b, compb));
         } else {
           // b was only partially filled by last read
           for (int k = 0; k < thisread; k++) {
-            assertTrue("file corrupted at or after byte " + (fileSize - bytesToRead), b[k]
== compb[k]);
+            assertTrue("file is corrupted at or after byte " +
+                (fileSize - bytesToRead), b[k] == compb[k]);
           }
         }
       }
-
-      // System.out.println("Read[" + i + "/" + readCount + "] " + thisread + " bytes.");
-
+      LOG.debug("Before update: to read: " + bytesToRead +
+          "; read already: "+ thisread);
       bytesToRead -= thisread;
-      i++;
+      LOG.debug("After  update: to read: " + bytesToRead +
+          "; read already: " + thisread);
     }
     stm.close();
   }
  
   /**
    * Test for block size of 2GB + 512B
+   * @throws IOException in case of errors
    */
   @Test
   public void testLargeBlockSize() throws IOException {
+    assumeTrue(ALLOWED_VM.equals(System.getProperty("sun.arch.data.model")));
     final long blockSize = 2L * 1024L * 1024L * 1024L + 512L; // 2GB + 512B
     runTest(blockSize);
   }
   
   /**
    * Test that we can write to and read from large blocks
+   * @param blockSize size of the block
+   * @throws IOException in case of errors
    */
   public void runTest(final long blockSize) throws IOException {
 
@@ -188,11 +191,12 @@ public class TestLargeBlock extends juni
     try {
 
       // create a new file in test data directory
-      Path file1 = new Path(System.getProperty("test.build.data") + "/" + Long.toString(blockSize)
+ ".dat");
+      Path file1 = new Path(System.getProperty("test.build.data") + "/" +
+          Long.toString(blockSize) + ".dat");
       FSDataOutputStream stm = createFile(fs, file1, 1, blockSize);
-      System.out.println("File " + file1 + " created with file size " +
-                         fileSize +
-                         " blocksize " + blockSize);
+      LOG.info("File " + file1 + " created with file size " +
+          fileSize +
+          " blocksize " + blockSize);
 
       // verify that file exists in FS namespace
       assertTrue(file1 + " should be a file", 
@@ -200,11 +204,11 @@ public class TestLargeBlock extends juni
 
       // write to file
       writeFile(stm, fileSize);
-      System.out.println("File " + file1 + " written to.");
+      LOG.info("File " + file1 + " written to.");
 
       // close file
       stm.close();
-      System.out.println("File " + file1 + " closed.");
+      LOG.info("File " + file1 + " closed.");
 
       // Make sure a client can read it
       checkFullFile(fs, file1, fileSize);



Mime
View raw message