hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject [3/3] hadoop git commit: HADOOP-13735 ITestS3AFileContextStatistics.testStatistics() failing. Contributed by Pieter Reuse
Date Thu, 20 Oct 2016 13:51:20 GMT
HADOOP-13735 ITestS3AFileContextStatistics.testStatistics() failing. Contributed by Pieter
Reuse


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ae270af
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ae270af
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ae270af

Branch: refs/heads/trunk
Commit: 9ae270af02c243993f853513c731cb268430e492
Parents: 73504b1
Author: Steve Loughran <stevel@apache.org>
Authored: Thu Oct 20 14:50:30 2016 +0100
Committer: Steve Loughran <stevel@apache.org>
Committed: Thu Oct 20 14:50:59 2016 +0100

----------------------------------------------------------------------
 .../java/org/apache/hadoop/fs/s3a/S3AFileSystem.java   |  2 +-
 .../org/apache/hadoop/fs/s3a/S3AInstrumentation.java   | 13 ++++++++++---
 2 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ae270af/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 2354819..9908ba7 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -548,7 +548,7 @@ public class S3AFileSystem extends FileSystem {
               progress,
               partSize,
               blockFactory,
-              instrumentation.newOutputStreamStatistics(),
+              instrumentation.newOutputStreamStatistics(statistics),
               new WriteOperationHelper(key)
           ),
           null);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ae270af/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
index 963c53f..fb8c852 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
@@ -37,6 +37,7 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.fs.FileSystem.Statistics;
 
 import static org.apache.hadoop.fs.s3a.Statistic.*;
 
@@ -639,9 +640,8 @@ public class S3AInstrumentation {
    * Create a stream output statistics instance.
    * @return the new instance
    */
-
-  OutputStreamStatistics newOutputStreamStatistics() {
-    return new OutputStreamStatistics();
+  OutputStreamStatistics newOutputStreamStatistics(Statistics statistics) {
+    return new OutputStreamStatistics(statistics);
   }
 
   /**
@@ -677,6 +677,12 @@ public class S3AInstrumentation {
     private final AtomicLong queueDuration = new AtomicLong(0);
     private final AtomicLong exceptionsInMultipartFinalize = new AtomicLong(0);
 
+    private Statistics statistics;
+
+    public OutputStreamStatistics(Statistics statistics){
+      this.statistics = statistics;
+    }
+
     /**
      * Block is queued for upload.
      */
@@ -717,6 +723,7 @@ public class S3AInstrumentation {
     /** Intermediate report of bytes uploaded. */
     void bytesTransferred(long byteCount) {
       bytesUploaded.addAndGet(byteCount);
+      statistics.incrementBytesWritten(byteCount);
       bytesPendingUpload.addAndGet(-byteCount);
       incrementGauge(STREAM_WRITE_BLOCK_UPLOADS_DATA_PENDING, -byteCount);
     }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message