Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 0959F18E00 for ; Fri, 10 Jul 2015 21:16:39 +0000 (UTC) Received: (qmail 29645 invoked by uid 500); 10 Jul 2015 21:16:38 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 29580 invoked by uid 500); 10 Jul 2015 21:16:38 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 29571 invoked by uid 99); 10 Jul 2015 21:16:38 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 10 Jul 2015 21:16:38 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A8ECDE363C; Fri, 10 Jul 2015 21:16:38 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jing9@apache.org To: common-commits@hadoop.apache.org Message-Id: <3244d3c166d6487b99b9a6069e67b408@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HDFS-8751. Remove setBlocks API from INodeFile and misc code cleanup. Contributed by Zhe Zhang Date: Fri, 10 Jul 2015 21:16:38 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/trunk 08244264c -> 47f4c5410 HDFS-8751. Remove setBlocks API from INodeFile and misc code cleanup. Contributed by Zhe Zhang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/47f4c541 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/47f4c541 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/47f4c541 Branch: refs/heads/trunk Commit: 47f4c54106ebb234a7d3dc71320aa584ecba161a Parents: 0824426 Author: Jing Zhao Authored: Fri Jul 10 14:15:22 2015 -0700 Committer: Jing Zhao Committed: Fri Jul 10 14:15:22 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hdfs/server/namenode/FSDirConcatOp.java | 2 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 33 +++++++++++++------- .../hdfs/server/namenode/TestINodeFile.java | 9 ++++++ 4 files changed, 34 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/47f4c541/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 13b2621..4fa566d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -713,6 +713,9 @@ Release 2.8.0 - UNRELEASED HDFS-8726. Move protobuf files that define the client-sever protocols to hdfs-client. (wheat9) + HDFS-8751. Remove setBlocks API from INodeFile and misc code cleanup. (Zhe + Zhang via jing9) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/47f4c541/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java index 3f22f51..bb00130 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java @@ -228,7 +228,7 @@ class FSDirConcatOp { int count = 0; for (INodeFile nodeToRemove : srcList) { if(nodeToRemove != null) { - nodeToRemove.setBlocks(null); + nodeToRemove.clearBlocks(); nodeToRemove.getParent().removeChild(nodeToRemove); fsd.getINodeMap().remove(nodeToRemove); count++; http://git-wip-us.apache.org/repos/asf/hadoop/blob/47f4c541/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index 48879d7..ec0c6c2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -369,7 +369,7 @@ public class INodeFile extends INodeWithAdditionalFields } /** Set the replication factor of this file. */ - public final void setFileReplication(short replication) { + private void setFileReplication(short replication) { header = HeaderFormat.REPLICATION.BITS.combine(replication, header); } @@ -413,33 +413,35 @@ public class INodeFile extends INodeWithAdditionalFields setStoragePolicyID(storagePolicyId); } - @Override + @Override // INodeFileAttributes public long getHeaderLong() { return header; } /** @return the blocks of the file. */ - @Override + @Override // BlockCollection public BlockInfo[] getBlocks() { return this.blocks; } /** @return blocks of the file corresponding to the snapshot. */ public BlockInfo[] getBlocks(int snapshot) { - if(snapshot == CURRENT_STATE_ID || getDiffs() == null) + if(snapshot == CURRENT_STATE_ID || getDiffs() == null) { return getBlocks(); + } FileDiff diff = getDiffs().getDiffById(snapshot); - BlockInfo[] snapshotBlocks = - diff == null ? getBlocks() : diff.getBlocks(); - if(snapshotBlocks != null) + BlockInfo[] snapshotBlocks = diff == null ? getBlocks() : diff.getBlocks(); + if (snapshotBlocks != null) { return snapshotBlocks; + } // Blocks are not in the current snapshot // Find next snapshot with blocks present or return current file blocks snapshotBlocks = getDiffs().findLaterSnapshotBlocks(snapshot); return (snapshotBlocks == null) ? getBlocks() : snapshotBlocks; } - void updateBlockCollection() { + /** Used during concat to update the BlockCollection for each block. */ + private void updateBlockCollection() { if (blocks != null) { for(BlockInfo b : blocks) { b.setBlockCollection(this); @@ -486,10 +488,15 @@ public class INodeFile extends INodeWithAdditionalFields } /** Set the blocks. */ - public void setBlocks(BlockInfo[] blocks) { + private void setBlocks(BlockInfo[] blocks) { this.blocks = blocks; } + /** Clear all blocks of the file. */ + public void clearBlocks() { + setBlocks(null); + } + @Override public void cleanSubtree(ReclaimContext reclaimContext, final int snapshot, int priorSnapshotId) { @@ -543,7 +550,7 @@ public class INodeFile extends INodeWithAdditionalFields blk.setBlockCollection(null); } } - setBlocks(null); + clearBlocks(); if (getAclFeature() != null) { AclStorage.removeAclFeature(getAclFeature()); } @@ -783,16 +790,18 @@ public class INodeFile extends INodeWithAdditionalFields public long collectBlocksBeyondMax(final long max, final BlocksMapUpdateInfo collectedBlocks) { final BlockInfo[] oldBlocks = getBlocks(); - if (oldBlocks == null) + if (oldBlocks == null) { return 0; + } // find the minimum n such that the size of the first n blocks > max int n = 0; long size = 0; for(; n < oldBlocks.length && max > size; n++) { size += oldBlocks[n].getNumBytes(); } - if (n >= oldBlocks.length) + if (n >= oldBlocks.length) { return size; + } // starting from block n, the data is beyond max. // resize the array. http://git-wip-us.apache.org/repos/asf/hadoop/blob/47f4c541/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java index ec4de1b..b8db998 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNull; import static org.junit.Assert.fail; import java.io.FileNotFoundException; @@ -1142,4 +1143,12 @@ public class TestINodeFile { f1 = inf.getXAttrFeature(); assertEquals(f1, null); } + + @Test + public void testClearBlocks() { + INodeFile toBeCleared = createINodeFiles(1, "toBeCleared")[0]; + assertEquals(1, toBeCleared.getBlocks().length); + toBeCleared.clearBlocks(); + assertNull(toBeCleared.getBlocks()); + } }