hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject [01/50] [abbrv] git commit: Merge branch 'trunk' into HDFS-6581
Date Wed, 01 Oct 2014 16:05:52 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 3f25d916d -> 04b08431a


Merge branch 'trunk' into HDFS-6581

Conflicts:
	hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
	hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e0d7fb48
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e0d7fb48
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e0d7fb48

Branch: refs/heads/trunk
Commit: e0d7fb48fbe872ef47803ceecdae654ab8c9d8af
Parents: a7814e1 ffdb7eb
Author: arp <arp@apache.org>
Authored: Tue Sep 16 14:32:57 2014 -0700
Committer: arp <arp@apache.org>
Committed: Tue Sep 16 14:32:57 2014 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-auth/pom.xml       |  13 +
 .../server/AuthenticationFilter.java            | 152 ++++--
 .../util/RandomSignerSecretProvider.java        |   4 +-
 .../util/RolloverSignerSecretProvider.java      |   7 +-
 .../util/SignerSecretProvider.java              |   9 +-
 .../util/StringSignerSecretProvider.java        |  15 +-
 .../util/ZKSignerSecretProvider.java            | 506 +++++++++++++++++++
 .../src/site/apt/Configuration.apt.vm           | 148 +++++-
 .../hadoop-auth/src/site/apt/index.apt.vm       |   5 +
 .../server/TestAuthenticationFilter.java        | 117 ++++-
 .../util/TestJaasConfiguration.java             |  55 ++
 .../util/TestRandomSignerSecretProvider.java    |   2 +-
 .../util/TestRolloverSignerSecretProvider.java  |   2 +-
 .../authentication/util/TestSigner.java         |  23 +-
 .../util/TestStringSignerSecretProvider.java    |   9 +-
 .../util/TestZKSignerSecretProvider.java        | 270 ++++++++++
 hadoop-common-project/hadoop-common/CHANGES.txt |   8 +
 .../src/main/bin/hadoop-functions.sh            | 118 +++--
 .../hadoop/fs/http/server/TestHttpFSServer.java |   8 +-
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |   6 +
 .../org/apache/hadoop/hdfs/DFSOutputStream.java |  40 +-
 .../datatransfer/DataTransferProtoUtil.java     |  44 +-
 .../hdfs/protocol/datatransfer/Receiver.java    | 125 +++--
 .../hdfs/protocol/datatransfer/Sender.java      |  26 +-
 .../BlockPlacementPolicyDefault.java            |   2 +-
 .../src/main/proto/datatransfer.proto           |   8 +
 .../blockmanagement/TestBlockManager.java       |  55 ++
 .../org/apache/hadoop/tracing/TestTracing.java  |  52 +-
 .../TestTracingShortCircuitLocalRead.java       |  97 ++++
 hadoop-project/pom.xml                          |  11 +
 .../org/apache/hadoop/fs/s3a/Constants.java     |  36 +-
 .../org/apache/hadoop/fs/s3a/S3AFileSystem.java |  46 +-
 .../apache/hadoop/fs/s3a/S3AOutputStream.java   |   6 +-
 hadoop-yarn-project/CHANGES.txt                 |   7 +
 .../hadoop/yarn/conf/YarnConfiguration.java     |  10 +
 .../applications/distributedshell/Client.java   |  18 +-
 .../TestDSSleepingAppMaster.java                |  58 +++
 .../distributedshell/TestDistributedShell.java  |  76 +++
 .../src/main/resources/yarn-default.xml         |  10 +
 .../util/CgroupsLCEResourcesHandler.java        |  29 +-
 .../util/TestCgroupsLCEResourcesHandler.java    | 165 +++++-
 41 files changed, 2117 insertions(+), 281 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0d7fb48/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index ed1abd8,d368f4e..6ff306c
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@@ -359,10 -364,17 +365,18 @@@ public class DFSOutputStream extends FS
      /**
       * Default construction for file create
       */
 -    private DataStreamer() {
 -      this(null);
 +    private DataStreamer(HdfsFileStatus stat) {
++      this(stat, null);
+     }
+ 
+     /**
+      * construction with tracing info
+      */
 -    private DataStreamer(Span span) {
++    private DataStreamer(HdfsFileStatus stat, Span span) {
        isAppend = false;
 +      isLazyPersistFile = stat.isLazyPersist();
        stage = BlockConstructionStage.PIPELINE_SETUP_CREATE;
+       traceSpan = span;
      }
      
      /**
@@@ -1614,7 -1633,11 +1636,11 @@@
      computePacketChunkSize(dfsClient.getConf().writePacketSize,
          checksum.getBytesPerChecksum());
  
-     streamer = new DataStreamer(stat);
+     Span traceSpan = null;
+     if (Trace.isTracing()) {
+       traceSpan = Trace.startSpan(this.getClass().getSimpleName()).detach();
+     }
 -    streamer = new DataStreamer(traceSpan);
++    streamer = new DataStreamer(stat, traceSpan);
      if (favoredNodes != null && favoredNodes.length != 0) {
        streamer.setFavoredNodes(favoredNodes);
      }
@@@ -1663,7 -1692,7 +1695,7 @@@
      } else {
        computePacketChunkSize(dfsClient.getConf().writePacketSize,
            checksum.getBytesPerChecksum());
-       streamer = new DataStreamer(stat);
 -      streamer = new DataStreamer(traceSpan);
++      streamer = new DataStreamer(stat, traceSpan);
      }
      this.fileEncryptionInfo = stat.getFileEncryptionInfo();
    }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0d7fb48/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
index 78693bb,daae9b7..8192925
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
@@@ -123,22 -131,27 +131,28 @@@ public abstract class Receiver implemen
    private void opWriteBlock(DataInputStream in) throws IOException {
      final OpWriteBlockProto proto = OpWriteBlockProto.parseFrom(vintPrefixed(in));
      final DatanodeInfo[] targets = PBHelper.convert(proto.getTargetsList());
-     writeBlock(PBHelper.convert(proto.getHeader().getBaseHeader().getBlock()),
-         PBHelper.convertStorageType(proto.getStorageType()),
-         PBHelper.convert(proto.getHeader().getBaseHeader().getToken()),
-         proto.getHeader().getClientName(),
-         targets,
-         PBHelper.convertStorageTypes(proto.getTargetStorageTypesList(), targets.length),
-         PBHelper.convert(proto.getSource()),
-         fromProto(proto.getStage()),
-         proto.getPipelineSize(),
-         proto.getMinBytesRcvd(), proto.getMaxBytesRcvd(),
-         proto.getLatestGenerationStamp(),
-         fromProto(proto.getRequestedChecksum()),
-         (proto.hasCachingStrategy() ?
-             getCachingStrategy(proto.getCachingStrategy()) :
-           CachingStrategy.newDefaultStrategy()),
-         (proto.hasAllowLazyPersist() ? proto.getAllowLazyPersist() : false));
+     TraceScope traceScope = continueTraceSpan(proto.getHeader(),
+         proto.getClass().getSimpleName());
+     try {
+       writeBlock(PBHelper.convert(proto.getHeader().getBaseHeader().getBlock()),
+           PBHelper.convertStorageType(proto.getStorageType()),
+           PBHelper.convert(proto.getHeader().getBaseHeader().getToken()),
+           proto.getHeader().getClientName(),
+           targets,
+           PBHelper.convertStorageTypes(proto.getTargetStorageTypesList(), targets.length),
+           PBHelper.convert(proto.getSource()),
+           fromProto(proto.getStage()),
+           proto.getPipelineSize(),
+           proto.getMinBytesRcvd(), proto.getMaxBytesRcvd(),
+           proto.getLatestGenerationStamp(),
+           fromProto(proto.getRequestedChecksum()),
+           (proto.hasCachingStrategy() ?
+               getCachingStrategy(proto.getCachingStrategy()) :
 -            CachingStrategy.newDefaultStrategy()));
 -     } finally {
 -      if (traceScope != null) traceScope.close();
 -     }
++            CachingStrategy.newDefaultStrategy()),
++          (proto.hasAllowLazyPersist() ? proto.getAllowLazyPersist() : false));
++    } finally {
++     if (traceScope != null) traceScope.close();
++    }
    }
  
    /** Receive {@link Op#TRANSFER_BLOCK} */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0d7fb48/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0d7fb48/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto
----------------------------------------------------------------------


Mime
View raw message