Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 49CCC18118 for ; Wed, 30 Sep 2015 18:23:38 +0000 (UTC) Received: (qmail 48891 invoked by uid 500); 30 Sep 2015 18:22:38 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 48645 invoked by uid 500); 30 Sep 2015 18:22:38 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 46610 invoked by uid 99); 30 Sep 2015 18:22:37 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 30 Sep 2015 18:22:37 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 4FFFDE056D; Wed, 30 Sep 2015 18:22:37 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aengineer@apache.org To: common-commits@hadoop.apache.org Date: Wed, 30 Sep 2015 18:23:25 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [51/52] [abbrv] hadoop git commit: Merge remote-tracking branch 'apache/trunk' into HDFS-7285 Merge remote-tracking branch 'apache/trunk' into HDFS-7285 Conflicts: hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java Change-Id: I53ec1c426dc988d6c4a2c87b00caef49c4057010 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6c17d315 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6c17d315 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6c17d315 Branch: refs/heads/HDFS-7240 Commit: 6c17d315287020368689fa078a40a1eaedf89d5b Parents: 8fd5520 854d25b Author: Zhe Zhang Authored: Wed Sep 30 08:39:13 2015 -0700 Committer: Zhe Zhang Committed: Wed Sep 30 08:39:13 2015 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 5 + .../org/apache/hadoop/fs/FSOutputSummer.java | 3 - .../main/java/org/apache/hadoop/util/GSet.java | 14 ++ .../org/apache/hadoop/util/GSetByHashMap.java | 6 + .../org/apache/hadoop/util/LightWeightGSet.java | 82 ++++-- .../hadoop/util/LightWeightResizableGSet.java | 129 ++++++++++ .../hadoop-common/src/site/markdown/Tracing.md | 15 +- .../apache/hadoop/fs/shell/TestTextCommand.java | 15 +- .../java/org/apache/hadoop/util/TestGSet.java | 69 ++++- .../hadoop/util/TestLightWeightCache.java | 6 + .../util/TestLightWeightResizableGSet.java | 252 +++++++++++++++++++ .../apache/hadoop/hdfs/BlockReaderFactory.java | 114 +++------ .../apache/hadoop/hdfs/BlockReaderLocal.java | 77 ++---- .../hadoop/hdfs/BlockReaderLocalLegacy.java | 20 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 36 +-- .../org/apache/hadoop/hdfs/DFSInputStream.java | 85 +++---- .../org/apache/hadoop/hdfs/DFSOutputStream.java | 36 +-- .../org/apache/hadoop/hdfs/DFSUtilClient.java | 10 +- .../org/apache/hadoop/hdfs/DataStreamer.java | 40 +-- .../apache/hadoop/hdfs/RemoteBlockReader.java | 4 +- .../apache/hadoop/hdfs/RemoteBlockReader2.java | 25 +- .../hdfs/client/HdfsClientConfigKeys.java | 2 - .../hadoop/hdfs/client/impl/LeaseRenewer.java | 18 +- .../protocol/datatransfer/PacketReceiver.java | 8 +- .../hdfs/protocol/datatransfer/Sender.java | 6 +- .../datatransfer/sasl/DataTransferSaslUtil.java | 8 +- .../ClientDatanodeProtocolTranslatorPB.java | 10 +- .../hdfs/shortcircuit/DfsClientShmManager.java | 45 +--- .../hdfs/shortcircuit/ShortCircuitCache.java | 101 ++------ .../hdfs/shortcircuit/ShortCircuitReplica.java | 33 +-- .../hdfs/shortcircuit/ShortCircuitShm.java | 14 +- .../hadoop/hdfs/util/ByteArrayManager.java | 63 +---- .../org/apache/hadoop/hdfs/web/TokenAspect.java | 8 +- .../hadoop/hdfs/web/URLConnectionFactory.java | 4 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 28 +-- .../services/org.apache.hadoop.fs.FileSystem | 1 + hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 21 ++ .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 9 +- .../java/org/apache/hadoop/hdfs/DFSUtil.java | 37 +-- .../hadoop/hdfs/server/datanode/DataNode.java | 11 +- .../hdfs/server/datanode/DirectoryScanner.java | 8 - .../hdfs/server/datanode/ReplicaInfo.java | 27 +- .../datanode/fsdataset/impl/BlockPoolSlice.java | 7 +- .../datanode/fsdataset/impl/ReplicaMap.java | 38 +-- .../server/datanode/web/DatanodeHttpServer.java | 15 ++ .../hdfs/server/namenode/NameNodeRpcServer.java | 2 +- .../hdfs/server/namenode/NamenodeFsck.java | 5 +- .../services/org.apache.hadoop.fs.FileSystem | 16 -- .../src/main/resources/hdfs-default.xml | 8 - .../org/apache/hadoop/hdfs/TestDFSUtil.java | 144 ++++++++--- .../blockmanagement/TestBlockTokenWithDFS.java | 2 + .../datanode/TestDataNodeVolumeFailure.java | 2 + .../apache/hadoop/tracing/TestTraceAdmin.java | 8 +- hadoop-yarn-project/CHANGES.txt | 6 + .../localizer/LocalResourcesTracker.java | 3 +- .../localizer/LocalResourcesTrackerImpl.java | 24 +- .../localizer/ResourceLocalizationService.java | 5 +- .../TestLocalResourcesTrackerImpl.java | 65 ++++- .../TestResourceLocalizationService.java | 16 +- .../scheduler/fair/QueueManager.java | 80 ++++-- 60 files changed, 1157 insertions(+), 794 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index be346a4,8f3df81..183602a --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@@ -3103,48 -3049,9 +3091,46 @@@ public class DFSClient implements java. } }); HEDGED_READ_THREAD_POOL.allowCoreThreadTimeOut(true); - if (LOG.isDebugEnabled()) { - LOG.debug("Using hedged reads; pool threads=" + num); - } + LOG.debug("Using hedged reads; pool threads={}", num); } + /** + * Create thread pool for parallel reading in striped layout, + * STRIPED_READ_THREAD_POOL, if it does not already exist. + * @param num Number of threads for striped reads thread pool. + */ + private void initThreadsNumForStripedReads(int num) { + assert num > 0; + if (STRIPED_READ_THREAD_POOL != null) { + return; + } + synchronized (DFSClient.class) { + if (STRIPED_READ_THREAD_POOL == null) { + STRIPED_READ_THREAD_POOL = new ThreadPoolExecutor(1, num, 60, + TimeUnit.SECONDS, new SynchronousQueue(), + new Daemon.DaemonFactory() { + private final AtomicInteger threadIndex = new AtomicInteger(0); + + @Override + public Thread newThread(Runnable r) { + Thread t = super.newThread(r); + t.setName("stripedRead-" + threadIndex.getAndIncrement()); + return t; + } + }, new ThreadPoolExecutor.CallerRunsPolicy() { + @Override + public void rejectedExecution(Runnable runnable, ThreadPoolExecutor e) { + LOG.info("Execution for striped reading rejected, " + + "Executing in current thread"); + // will run in the current thread + super.rejectedExecution(runnable, e); + } + }); + STRIPED_READ_THREAD_POOL.allowCoreThreadTimeOut(true); + } + } + } + ThreadPoolExecutor getHedgedReadsThreadPool() { return HEDGED_READ_THREAD_POOL; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index 78eaa6c,47c4b7e..6039177 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@@ -191,11 -190,11 +191,11 @@@ public class DFSOutputStream extends FS this.fileEncryptionInfo = stat.getFileEncryptionInfo(); this.cachingStrategy = new AtomicReference( dfsClient.getDefaultWriteCachingStrategy()); - if ((progress != null) && DFSClient.LOG.isDebugEnabled()) { - DFSClient.LOG.debug( - "Set non-null progress callback on DFSOutputStream " + src); + if (progress != null) { + DFSClient.LOG.debug("Set non-null progress callback on DFSOutputStream " + +"{}", src); } - + this.bytesPerChecksum = checksum.getBytesPerChecksum(); if (bytesPerChecksum <= 0) { throw new HadoopIllegalArgumentException( @@@ -410,14 -394,10 +407,10 @@@ if (currentPacket == null) { currentPacket = createPacket(packetSize, chunksPerPacket, getStreamer() .getBytesCurBlock(), getStreamer().getAndIncCurrentSeqno(), false); - if (LOG.isDebugEnabled()) { - LOG.debug("WriteChunk allocating new packet seqno=" + - currentPacket.getSeqno() + - ", src=" + src + - ", packetSize=" + packetSize + - ", chunksPerPacket=" + chunksPerPacket + - ", bytesCurBlock=" + getStreamer().getBytesCurBlock() + ", " + this); - } - DFSClient.LOG.debug("DFSClient writeChunk allocating new packet seqno={}," ++ DFSClient.LOG.debug("WriteChunk allocating new packet seqno={}," + + " src={}, packetSize={}, chunksPerPacket={}, bytesCurBlock={}", + currentPacket.getSeqno(), src, packetSize, chunksPerPacket, - getStreamer().getBytesCurBlock()); ++ getStreamer().getBytesCurBlock() + ", " + this); } currentPacket.writeChecksum(checksum, ckoff, cklen); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java index af7a61e,4ea1f41..683d98d --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java @@@ -507,10 -479,8 +503,8 @@@ class DataStreamer extends Daemon stage = BlockConstructionStage.DATA_STREAMING; } - private void endBlock() { - LOG.debug("Closing old block {}", block); + protected void endBlock() { - if(LOG.isDebugEnabled()) { - LOG.debug("Closing old block " + block); - } ++ LOG.debug("Closing old block " + block); this.setName("DataStreamer for file " + src); closeResponder(); closeStream(); @@@ -590,10 -560,8 +584,11 @@@ } // get new block from namenode. + if (LOG.isDebugEnabled()) { + LOG.debug("stage=" + stage + ", " + this); + } if (stage == BlockConstructionStage.PIPELINE_SETUP_CREATE) { + LOG.debug("Allocating new block"); setPipeline(nextBlockOutputStream()); initDataStreaming(); } else if (stage == BlockConstructionStage.PIPELINE_SETUP_APPEND) { @@@ -645,9 -617,7 +641,7 @@@ } } - if (LOG.isDebugEnabled()) { - LOG.debug(this + " sending " + one); - } - LOG.debug("DataStreamer block {} sending packet {}", block, one); ++ LOG.debug(this + " sending " + one); // write out data to remote datanode TraceScope writeScope = dfsClient.getTracer(). @@@ -1785,9 -1768,7 +1773,7 @@@ packet.addTraceParent(Tracer.getCurrentSpanId()); dataQueue.addLast(packet); lastQueuedSeqno = packet.getSeqno(); - if (LOG.isDebugEnabled()) { - LOG.debug("Queued " + packet + ", " + this); - } - LOG.debug("Queued packet {}", packet.getSeqno()); ++ LOG.debug("Queued " + packet + ", " + this); dataQueue.notifyAll(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader2.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java ----------------------------------------------------------------------