Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: rakeshr@apache.org
To: common-commits@hadoop.apache.org
Date: Tue, 11 Jul 2017 16:25:32 -0000
Message-Id: <305bf0d3d8174f7dba511683a175f532@git.apache.org>
In-Reply-To: <338233d9fb704c7586da676be7e68fa8@git.apache.org>
References: <338233d9fb704c7586da676be7e68fa8@git.apache.org>
Subject: [46/50] [abbrv] hadoop git commit: HDFS-11762. [SPS]: Empty files
 should be ignored in StoragePolicySatisfier. Surendra Singh Lilhore.
archived-at: Tue, 11 Jul 2017 16:25:06 -0000

HDFS-11762. [SPS]: Empty files should be ignored in StoragePolicySatisfier. Surendra Singh Lilhore.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ecb55713
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ecb55713
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ecb55713

Branch: refs/heads/HDFS-10285
Commit: ecb5571337aba985045973b4712b7ae0adb7a6ae
Parents: c1eeb6c
Author: Rakesh Radhakrishnan <rakeshr@apache.org>
Authored: Mon Jun 5 12:32:41 2017 +0530
Committer: Rakesh Radhakrishnan <rakeshr@apache.org>
Committed: Tue Jul 11 21:04:50 2017 +0530

----------------------------------------------------------------------
 .../namenode/FSDirSatisfyStoragePolicyOp.java   | 15 ++++++---
 .../namenode/TestStoragePolicySatisfier.java    | 32 ++++++++++++++++++++
 2 files changed, 42 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecb55713/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSatisfyStoragePolicyOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSatisfyStoragePolicyOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSatisfyStoragePolicyOp.java
index 81d337f..bd4e5ed 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSatisfyStoragePolicyOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSatisfyStoragePolicyOp.java
@@ -51,7 +51,6 @@ final class FSDirSatisfyStoragePolicyOp {
 
     assert fsd.getFSNamesystem().hasWriteLock();
     FSPermissionChecker pc = fsd.getPermissionChecker();
-    List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
     INodesInPath iip;
     fsd.writeLock();
     try {
@@ -62,8 +61,11 @@ final class FSDirSatisfyStoragePolicyOp {
         fsd.checkPathAccess(pc, iip, FsAction.WRITE);
       }
       XAttr satisfyXAttr = unprotectedSatisfyStoragePolicy(iip, bm, fsd);
-      xAttrs.add(satisfyXAttr);
-      fsd.getEditLog().logSetXAttrs(src, xAttrs, logRetryCache);
+      if (satisfyXAttr != null) {
+        List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
+        xAttrs.add(satisfyXAttr);
+        fsd.getEditLog().logSetXAttrs(src, xAttrs, logRetryCache);
+      }
     } finally {
       fsd.writeUnlock();
     }
@@ -79,16 +81,19 @@ final class FSDirSatisfyStoragePolicyOp {
 
     // TODO: think about optimization here, label the dir instead
     // of the sub-files of the dir.
-    if (inode.isFile()) {
+    if (inode.isFile() && inode.asFile().numBlocks() != 0) {
       candidateNodes.add(inode);
     } else if (inode.isDirectory()) {
       for (INode node : inode.asDirectory().getChildrenList(snapshotId)) {
-        if (node.isFile()) {
+        if (node.isFile() && node.asFile().numBlocks() != 0) {
           candidateNodes.add(node);
         }
       }
     }
 
+    if (candidateNodes.isEmpty()) {
+      return null;
+    }
     // If node has satisfy xattr, then stop adding it
     // to satisfy movement queue.
     if (inodeHasSatisfyXAttr(candidateNodes)) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecb55713/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfier.java
index fa954b8..8e08a1e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfier.java
@@ -907,6 +907,38 @@ public class TestStoragePolicySatisfier {
     }
   }
 
+  /**
+   * Test SPS with empty file.
+   * 1. Create one empty file.
+   * 2. Call satisfyStoragePolicy for empty file.
+   * 3. SPS should skip this file and xattr should not be added for empty file.
+   */
+  @Test(timeout = 300000)
+  public void testSPSWhenFileLengthIsZero() throws Exception {
+    MiniDFSCluster cluster = null;
+    try {
+      cluster = new MiniDFSCluster.Builder(new Configuration()).numDataNodes(0)
+          .build();
+      cluster.waitActive();
+      DistributedFileSystem fs = cluster.getFileSystem();
+      Path filePath = new Path("/zeroSizeFile");
+      DFSTestUtil.createFile(fs, filePath, 0, (short) 1, 0);
+      FSEditLog editlog = cluster.getNameNode().getNamesystem().getEditLog();
+      long lastWrittenTxId = editlog.getLastWrittenTxId();
+      fs.satisfyStoragePolicy(filePath);
+      Assert.assertEquals("Xattr should not be added for the file",
+          lastWrittenTxId, editlog.getLastWrittenTxId());
+      INode inode = cluster.getNameNode().getNamesystem().getFSDirectory()
+          .getINode(filePath.toString());
+      Assert.assertTrue("XAttrFeature should be null for file",
+          inode.getXAttrFeature() == null);
+    } finally {
+      if (cluster != null) {
+        cluster.shutdown();
+      }
+    }
+  }
+
   private String createFileAndSimulateFavoredNodes(int favoredNodesCount)
       throws IOException {
     ArrayList<DataNode> dns = hdfsCluster.getDataNodes();


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org