Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: wwei@apache.org
To: common-commits@hadoop.apache.org
Message-Id: <a90eba214b9c4c4c8e3a12ae3d971eaa@git.apache.org>
Subject: hadoop git commit: HDFS-12298. Ozone: Block deletion service floods
 the log when deleting large number of block files. Contributed by Yiqun Lin.
Date: Tue, 15 Aug 2017 09:13:26 +0000 (UTC)
archived-at: Tue, 15 Aug 2017 09:13:31 -0000

Repository: hadoop
Updated Branches:
  refs/heads/HDFS-7240 e53567ba5 -> f7ae9f6e4


HDFS-12298. Ozone: Block deletion service floods the log when deleting large number of block files. Contributed by Yiqun Lin.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f7ae9f6e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f7ae9f6e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f7ae9f6e

Branch: refs/heads/HDFS-7240
Commit: f7ae9f6e41acb0e44449ffb72a7bcf2a1f151141
Parents: e53567b
Author: Weiwei Yang <wwei@apache.org>
Authored: Tue Aug 15 17:12:59 2017 +0800
Committer: Weiwei Yang <wwei@apache.org>
Committed: Tue Aug 15 17:12:59 2017 +0800

----------------------------------------------------------------------
 .../statemachine/background/BlockDeletingService.java | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ae9f6e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
index 618fa42..df59f68 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
 import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
 import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
 import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.utils.BackgroundService;
 import org.apache.hadoop.utils.BackgroundTaskResult;
 import org.apache.hadoop.utils.BackgroundTaskQueue;
@@ -157,6 +158,7 @@ public class BlockDeletingService extends BackgroundService{
 
     @Override
     public BackgroundTaskResult call() throws Exception {
+      long startTime = Time.monotonicNow();
       // Scan container's db and get list of under deletion blocks
       MetadataStore meta = KeyUtils.getDB(containerData, conf);
       // # of blocks to delete is throttled
@@ -165,16 +167,16 @@ public class BlockDeletingService extends BackgroundService{
       List<Map.Entry<byte[], byte[]>> toDeleteBlocks =
           meta.getRangeKVs(null, blockLimitPerTask, filter);
       if (toDeleteBlocks.isEmpty()) {
-        LOG.info("No under deletion block found in container : {}",
+        LOG.debug("No under deletion block found in container : {}",
             containerData.getContainerName());
       }
 
       List<String> succeedBlocks = new LinkedList<>();
-      LOG.info("Container : {}, To-Delete blocks : {}",
+      LOG.debug("Container : {}, To-Delete blocks : {}",
           containerData.getContainerName(), toDeleteBlocks.size());
       toDeleteBlocks.forEach(entry -> {
         String blockName = DFSUtil.bytes2String(entry.getKey());
-        LOG.info("Deleting block {}", blockName);
+        LOG.debug("Deleting block {}", blockName);
         try {
           ContainerProtos.KeyData data =
               ContainerProtos.KeyData.parseFrom(entry.getValue());
@@ -182,7 +184,7 @@ public class BlockDeletingService extends BackgroundService{
           for (ContainerProtos.ChunkInfo chunkInfo : data.getChunksList()) {
             File chunkFile = new File(chunkInfo.getChunkName());
             if (FileUtils.deleteQuietly(chunkFile)) {
-              LOG.info("block {} chunk {} deleted", blockName,
+              LOG.debug("block {} chunk {} deleted", blockName,
                   chunkFile.getAbsolutePath());
             }
           }
@@ -198,6 +200,10 @@ public class BlockDeletingService extends BackgroundService{
           batch.delete(DFSUtil.string2Bytes(entry)));
       meta.writeBatch(batch);
 
+      LOG.info("The elapsed time of task@{} for"
+          + " deleting blocks: {}ms.",
+          Integer.toHexString(this.hashCode()),
+          Time.monotonicNow() - startTime);
       ContainerBackgroundTaskResult crr = new ContainerBackgroundTaskResult();
       crr.addAll(succeedBlocks);
       return crr;


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org