hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cmcc...@apache.org
Subject svn commit: r1547118 - in /hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/fs/ src/test/java/org/apache/hadoop/fs/
Date Mon, 02 Dec 2013 17:29:46 GMT
Author: cmccabe
Date: Mon Dec  2 17:29:45 2013
New Revision: 1547118

URL: http://svn.apache.org/r1547118
Log:
HADOOP-10130. RawLocalFS pread does not track FileSystem Statistics (Binglin Chang via Colin
Patrick McCabe)

Modified:
    hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/CHANGES.txt
    hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
    hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
    hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFsFCStatistics.java

Modified: hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1547118&r1=1547117&r2=1547118&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/CHANGES.txt Mon
Dec  2 17:29:45 2013
@@ -64,6 +64,9 @@ Release 2.2.1 - UNRELEASED
     HADOOP-9114. After defined the dfs.checksum.type as the NULL, write file and hflush will

     through java.lang.ArrayIndexOutOfBoundsException (Sathish via umamahesh)
 
+    HADOOP-10130. RawLocalFS::LocalFSFileInputStream.pread does not track
+    FS::Statistics (Binglin Chang via Colin Patrick McCabe)
+
 Release 2.2.0 - 2013-10-13
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=1547118&r1=1547117&r2=1547118&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
(original)
+++ hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
Mon Dec  2 17:29:45 2013
@@ -82,39 +82,6 @@ public class RawLocalFileSystem extends 
     setConf(conf);
   }
   
-  class TrackingFileInputStream extends FileInputStream {
-    public TrackingFileInputStream(File f) throws IOException {
-      super(f);
-    }
-    
-    @Override
-    public int read() throws IOException {
-      int result = super.read();
-      if (result != -1) {
-        statistics.incrementBytesRead(1);
-      }
-      return result;
-    }
-    
-    @Override
-    public int read(byte[] data) throws IOException {
-      int result = super.read(data);
-      if (result != -1) {
-        statistics.incrementBytesRead(result);
-      }
-      return result;
-    }
-    
-    @Override
-    public int read(byte[] data, int offset, int length) throws IOException {
-      int result = super.read(data, offset, length);
-      if (result != -1) {
-        statistics.incrementBytesRead(result);
-      }
-      return result;
-    }
-  }
-
   /*******************************************************
    * For open()'s FSInputStream.
    *******************************************************/
@@ -123,7 +90,7 @@ public class RawLocalFileSystem extends 
     private long position;
 
     public LocalFSFileInputStream(Path f) throws IOException {
-      this.fis = new TrackingFileInputStream(pathToFile(f));
+      fis = new FileInputStream(pathToFile(f));
     }
     
     @Override
@@ -158,6 +125,7 @@ public class RawLocalFileSystem extends 
         int value = fis.read();
         if (value >= 0) {
           this.position++;
+          statistics.incrementBytesRead(1);
         }
         return value;
       } catch (IOException e) {                 // unexpected exception
@@ -171,6 +139,7 @@ public class RawLocalFileSystem extends 
         int value = fis.read(b, off, len);
         if (value > 0) {
           this.position += value;
+          statistics.incrementBytesRead(value);
         }
         return value;
       } catch (IOException e) {                 // unexpected exception
@@ -183,7 +152,11 @@ public class RawLocalFileSystem extends 
       throws IOException {
       ByteBuffer bb = ByteBuffer.wrap(b, off, len);
       try {
-        return fis.getChannel().read(bb, position);
+        int value = fis.getChannel().read(bb, position);
+        if (value > 0) {
+          statistics.incrementBytesRead(value);
+        }
+        return value;
       } catch (IOException e) {
         throw new FSError(e);
       }

Modified: hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java?rev=1547118&r1=1547117&r2=1547118&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
(original)
+++ hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
Mon Dec  2 17:29:45 2013
@@ -57,6 +57,7 @@ public abstract class FCStatisticsBaseTe
     FSDataInputStream fstr = fc.open(filePath);
     byte[] buf = new byte[blockSize];
     int bytesRead = fstr.read(buf, 0, blockSize);
+    fstr.read(0, buf, 0, blockSize);
     Assert.assertEquals(blockSize, bytesRead);
     verifyReadBytes(stats);
     verifyWrittenBytes(stats);

Modified: hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFsFCStatistics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFsFCStatistics.java?rev=1547118&r1=1547117&r2=1547118&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFsFCStatistics.java
(original)
+++ hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFsFCStatistics.java
Mon Dec  2 17:29:45 2013
@@ -47,7 +47,8 @@ public class TestLocalFsFCStatistics ext
 
   @Override
   protected void verifyReadBytes(Statistics stats) {
-    Assert.assertEquals(blockSize, stats.getBytesRead());
+    // one blockSize for read, one for pread
+    Assert.assertEquals(2*blockSize, stats.getBytesRead());
   }
 
   @Override



Mime
View raw message