hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject [50/50] [abbrv] hadoop git commit: Merge remote-tracking branch 'apache/trunk' into HDFS-7285
Date Wed, 30 Sep 2015 15:42:50 GMT
Merge remote-tracking branch 'apache/trunk' into HDFS-7285

Conflicts:
	hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
	hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java

Change-Id: I53ec1c426dc988d6c4a2c87b00caef49c4057010


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6c17d315
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6c17d315
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6c17d315

Branch: refs/heads/trunk
Commit: 6c17d315287020368689fa078a40a1eaedf89d5b
Parents: 8fd5520 854d25b
Author: Zhe Zhang <zhezhang@cloudera.com>
Authored: Wed Sep 30 08:39:13 2015 -0700
Committer: Zhe Zhang <zhezhang@cloudera.com>
Committed: Wed Sep 30 08:39:13 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |   5 +
 .../org/apache/hadoop/fs/FSOutputSummer.java    |   3 -
 .../main/java/org/apache/hadoop/util/GSet.java  |  14 ++
 .../org/apache/hadoop/util/GSetByHashMap.java   |   6 +
 .../org/apache/hadoop/util/LightWeightGSet.java |  82 ++++--
 .../hadoop/util/LightWeightResizableGSet.java   | 129 ++++++++++
 .../hadoop-common/src/site/markdown/Tracing.md  |  15 +-
 .../apache/hadoop/fs/shell/TestTextCommand.java |  15 +-
 .../java/org/apache/hadoop/util/TestGSet.java   |  69 ++++-
 .../hadoop/util/TestLightWeightCache.java       |   6 +
 .../util/TestLightWeightResizableGSet.java      | 252 +++++++++++++++++++
 .../apache/hadoop/hdfs/BlockReaderFactory.java  | 114 +++------
 .../apache/hadoop/hdfs/BlockReaderLocal.java    |  77 ++----
 .../hadoop/hdfs/BlockReaderLocalLegacy.java     |  20 +-
 .../java/org/apache/hadoop/hdfs/DFSClient.java  |  36 +--
 .../org/apache/hadoop/hdfs/DFSInputStream.java  |  85 +++----
 .../org/apache/hadoop/hdfs/DFSOutputStream.java |  36 +--
 .../org/apache/hadoop/hdfs/DFSUtilClient.java   |  10 +-
 .../org/apache/hadoop/hdfs/DataStreamer.java    |  40 +--
 .../apache/hadoop/hdfs/RemoteBlockReader.java   |   4 +-
 .../apache/hadoop/hdfs/RemoteBlockReader2.java  |  25 +-
 .../hdfs/client/HdfsClientConfigKeys.java       |   2 -
 .../hadoop/hdfs/client/impl/LeaseRenewer.java   |  18 +-
 .../protocol/datatransfer/PacketReceiver.java   |   8 +-
 .../hdfs/protocol/datatransfer/Sender.java      |   6 +-
 .../datatransfer/sasl/DataTransferSaslUtil.java |   8 +-
 .../ClientDatanodeProtocolTranslatorPB.java     |  10 +-
 .../hdfs/shortcircuit/DfsClientShmManager.java  |  45 +---
 .../hdfs/shortcircuit/ShortCircuitCache.java    | 101 ++------
 .../hdfs/shortcircuit/ShortCircuitReplica.java  |  33 +--
 .../hdfs/shortcircuit/ShortCircuitShm.java      |  14 +-
 .../hadoop/hdfs/util/ByteArrayManager.java      |  63 +----
 .../org/apache/hadoop/hdfs/web/TokenAspect.java |   8 +-
 .../hadoop/hdfs/web/URLConnectionFactory.java   |   4 +-
 .../hadoop/hdfs/web/WebHdfsFileSystem.java      |  28 +--
 .../services/org.apache.hadoop.fs.FileSystem    |   1 +
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  21 ++
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |   9 +-
 .../java/org/apache/hadoop/hdfs/DFSUtil.java    |  37 +--
 .../hadoop/hdfs/server/datanode/DataNode.java   |  11 +-
 .../hdfs/server/datanode/DirectoryScanner.java  |   8 -
 .../hdfs/server/datanode/ReplicaInfo.java       |  27 +-
 .../datanode/fsdataset/impl/BlockPoolSlice.java |   7 +-
 .../datanode/fsdataset/impl/ReplicaMap.java     |  38 +--
 .../server/datanode/web/DatanodeHttpServer.java |  15 ++
 .../hdfs/server/namenode/NameNodeRpcServer.java |   2 +-
 .../hdfs/server/namenode/NamenodeFsck.java      |   5 +-
 .../services/org.apache.hadoop.fs.FileSystem    |  16 --
 .../src/main/resources/hdfs-default.xml         |   8 -
 .../org/apache/hadoop/hdfs/TestDFSUtil.java     | 144 ++++++++---
 .../blockmanagement/TestBlockTokenWithDFS.java  |   2 +
 .../datanode/TestDataNodeVolumeFailure.java     |   2 +
 .../apache/hadoop/tracing/TestTraceAdmin.java   |   8 +-
 hadoop-yarn-project/CHANGES.txt                 |   6 +
 .../localizer/LocalResourcesTracker.java        |   3 +-
 .../localizer/LocalResourcesTrackerImpl.java    |  24 +-
 .../localizer/ResourceLocalizationService.java  |   5 +-
 .../TestLocalResourcesTrackerImpl.java          |  65 ++++-
 .../TestResourceLocalizationService.java        |  16 +-
 .../scheduler/fair/QueueManager.java            |  80 ++++--
 60 files changed, 1157 insertions(+), 794 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index be346a4,8f3df81..183602a
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@@ -3103,48 -3049,9 +3091,46 @@@ public class DFSClient implements java.
        }
      });
      HEDGED_READ_THREAD_POOL.allowCoreThreadTimeOut(true);
-     if (LOG.isDebugEnabled()) {
-       LOG.debug("Using hedged reads; pool threads=" + num);
-     }
+     LOG.debug("Using hedged reads; pool threads={}", num);
    }
  
 +  /**
 +   * Create thread pool for parallel reading in striped layout,
 +   * STRIPED_READ_THREAD_POOL, if it does not already exist.
 +   * @param num Number of threads for striped reads thread pool.
 +   */
 +  private void initThreadsNumForStripedReads(int num) {
 +    assert num > 0;
 +    if (STRIPED_READ_THREAD_POOL != null) {
 +      return;
 +    }
 +    synchronized (DFSClient.class) {
 +      if (STRIPED_READ_THREAD_POOL == null) {
 +        STRIPED_READ_THREAD_POOL = new ThreadPoolExecutor(1, num, 60,
 +            TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
 +            new Daemon.DaemonFactory() {
 +          private final AtomicInteger threadIndex = new AtomicInteger(0);
 +
 +          @Override
 +          public Thread newThread(Runnable r) {
 +            Thread t = super.newThread(r);
 +            t.setName("stripedRead-" + threadIndex.getAndIncrement());
 +            return t;
 +          }
 +        }, new ThreadPoolExecutor.CallerRunsPolicy() {
 +          @Override
 +          public void rejectedExecution(Runnable runnable, ThreadPoolExecutor e) {
 +            LOG.info("Execution for striped reading rejected, "
 +                + "Executing in current thread");
 +            // will run in the current thread
 +            super.rejectedExecution(runnable, e);
 +          }
 +        });
 +        STRIPED_READ_THREAD_POOL.allowCoreThreadTimeOut(true);
 +      }
 +    }
 +  }
 +
    ThreadPoolExecutor getHedgedReadsThreadPool() {
      return HEDGED_READ_THREAD_POOL;
    }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index 78eaa6c,47c4b7e..6039177
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@@ -191,11 -190,11 +191,11 @@@ public class DFSOutputStream extends FS
      this.fileEncryptionInfo = stat.getFileEncryptionInfo();
      this.cachingStrategy = new AtomicReference<CachingStrategy>(
          dfsClient.getDefaultWriteCachingStrategy());
-     if ((progress != null) && DFSClient.LOG.isDebugEnabled()) {
-       DFSClient.LOG.debug(
-           "Set non-null progress callback on DFSOutputStream " + src);
+     if (progress != null) {
+       DFSClient.LOG.debug("Set non-null progress callback on DFSOutputStream "
+           +"{}", src);
      }
 -    
 +
      this.bytesPerChecksum = checksum.getBytesPerChecksum();
      if (bytesPerChecksum <= 0) {
        throw new HadoopIllegalArgumentException(
@@@ -410,14 -394,10 +407,10 @@@
      if (currentPacket == null) {
        currentPacket = createPacket(packetSize, chunksPerPacket, getStreamer()
            .getBytesCurBlock(), getStreamer().getAndIncCurrentSeqno(), false);
-       if (LOG.isDebugEnabled()) {
-         LOG.debug("WriteChunk allocating new packet seqno=" +
-             currentPacket.getSeqno() +
-             ", src=" + src +
-             ", packetSize=" + packetSize +
-             ", chunksPerPacket=" + chunksPerPacket +
-             ", bytesCurBlock=" + getStreamer().getBytesCurBlock() + ", " + this);
-       }
 -      DFSClient.LOG.debug("DFSClient writeChunk allocating new packet seqno={},"
++      DFSClient.LOG.debug("WriteChunk allocating new packet seqno={},"
+               + " src={}, packetSize={}, chunksPerPacket={}, bytesCurBlock={}",
+           currentPacket.getSeqno(), src, packetSize, chunksPerPacket,
 -          getStreamer().getBytesCurBlock());
++          getStreamer().getBytesCurBlock() + ", " + this);
      }
  
      currentPacket.writeChecksum(checksum, ckoff, cklen);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index af7a61e,4ea1f41..683d98d
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@@ -507,10 -479,8 +503,8 @@@ class DataStreamer extends Daemon 
      stage = BlockConstructionStage.DATA_STREAMING;
    }
  
 -  private void endBlock() {
 -    LOG.debug("Closing old block {}", block);
 +  protected void endBlock() {
-     if(LOG.isDebugEnabled()) {
-       LOG.debug("Closing old block " + block);
-     }
++    LOG.debug("Closing old block " + block);
      this.setName("DataStreamer for file " + src);
      closeResponder();
      closeStream();
@@@ -590,10 -560,8 +584,11 @@@
          }
  
          // get new block from namenode.
 +        if (LOG.isDebugEnabled()) {
 +          LOG.debug("stage=" + stage + ", " + this);
 +        }
          if (stage == BlockConstructionStage.PIPELINE_SETUP_CREATE) {
+           LOG.debug("Allocating new block");
            setPipeline(nextBlockOutputStream());
            initDataStreaming();
          } else if (stage == BlockConstructionStage.PIPELINE_SETUP_APPEND) {
@@@ -645,9 -617,7 +641,7 @@@
            }
          }
  
-         if (LOG.isDebugEnabled()) {
-           LOG.debug(this + " sending " + one);
-         }
 -        LOG.debug("DataStreamer block {} sending packet {}", block, one);
++        LOG.debug(this + " sending " + one);
  
          // write out data to remote datanode
          TraceScope writeScope = dfsClient.getTracer().
@@@ -1785,9 -1768,7 +1773,7 @@@
        packet.addTraceParent(Tracer.getCurrentSpanId());
        dataQueue.addLast(packet);
        lastQueuedSeqno = packet.getSeqno();
-       if (LOG.isDebugEnabled()) {
-         LOG.debug("Queued " + packet + ", " + this);
-       }
 -      LOG.debug("Queued packet {}", packet.getSeqno());
++      LOG.debug("Queued " + packet + ", " + this);
        dataQueue.notifyAll();
      }
    }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader2.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c17d315/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
----------------------------------------------------------------------


Mime
View raw message