hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ji...@apache.org
Subject git commit: HDFS-7167. NPE while running Mover if the given path is for a file. Contributed by Jing Zhao.
Date Tue, 30 Sep 2014 18:23:22 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 6a60fa279 -> 635fdcbe5


HDFS-7167. NPE while running Mover if the given path is for a file. Contributed by Jing Zhao.

Conflicts:
	hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/635fdcbe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/635fdcbe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/635fdcbe

Branch: refs/heads/branch-2
Commit: 635fdcbe5c330c31b5f38352a0732c529950ffd3
Parents: 6a60fa2
Author: Jing Zhao <jing9@apache.org>
Authored: Tue Sep 30 11:13:35 2014 -0700
Committer: Jing Zhao <jing9@apache.org>
Committed: Tue Sep 30 11:19:57 2014 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     | 10 +++--
 .../apache/hadoop/hdfs/server/mover/Mover.java  | 22 ++++-------
 .../hdfs/server/mover/TestStorageMover.java     | 40 ++++++++++++++++++--
 3 files changed, 50 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/635fdcbe/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ea4c00c..2cc76c9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -520,6 +520,11 @@ Release 2.6.0 - UNRELEASED
     HDFS-7157. Using Time.now() for recording start/end time of reconfiguration
     tasks (Lei Xu via cmccabe)
 
+    HDFS-6664. HDFS permissions guide documentation states incorrect default
+    group mapping class. (Ray Chiang via aw)
+
+    HDFS-4227. Document dfs.namenode.resource.*  (Daisuke Kobayashi via aw)
+
     BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
   
       HDFS-6387. HDFS CLI admin tool for creating & deleting an
@@ -751,10 +756,7 @@ Release 2.6.0 - UNRELEASED
     HDFS-7140. Add a tool to list all the existing block storage policies.
     (jing9)
 
-    HDFS-6664. HDFS permissions guide documentation states incorrect default
-    group mapping class. (Ray Chiang via aw)
-
-    HDFS-4227. Document dfs.namenode.resource.*  (Daisuke Kobayashi via aw)
+    HDFS-7167. NPE while running Mover if the given path is for a file. (jing9)
 
 Release 2.5.1 - 2014-09-05
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/635fdcbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
index 4db0df6..04133bd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
@@ -252,14 +252,9 @@ public class Mover {
      */
     private boolean processNamespace() {
       getSnapshottableDirs();
-      boolean hasRemaining = true;
-      try {
-        for (Path target : targetPaths) {
-          hasRemaining = processDirRecursively("", dfs.getFileInfo(target
-              .toUri().getPath()));
-        }
-      } catch (IOException e) {
-        LOG.warn("Failed to get root directory status. Ignore and continue.", e);
+      boolean hasRemaining = false;
+      for (Path target : targetPaths) {
+        hasRemaining |= processPath(target.toUri().getPath());
       }
       // wait for pending move to finish and retry the failed migration
       hasRemaining |= Dispatcher.waitForMoveCompletion(storages.targets.values());
@@ -270,7 +265,7 @@ public class Mover {
      * @return whether there is still remaing migration work for the next
      *         round
      */
-    private boolean processChildrenList(String fullPath) {
+    private boolean processPath(String fullPath) {
       boolean hasRemaining = false;
       for (byte[] lastReturnedName = HdfsFileStatus.EMPTY_NAME;;) {
         final DirectoryListing children;
@@ -285,7 +280,7 @@ public class Mover {
           return hasRemaining;
         }
         for (HdfsFileStatus child : children.getPartialListing()) {
-          hasRemaining |= processDirRecursively(fullPath, child);
+          hasRemaining |= processRecursively(fullPath, child);
         }
         if (children.hasMore()) {
           lastReturnedName = children.getLastName();
@@ -296,8 +291,7 @@ public class Mover {
     }
 
     /** @return whether the migration requires next round */
-    private boolean processDirRecursively(String parent,
-                                          HdfsFileStatus status) {
+    private boolean processRecursively(String parent, HdfsFileStatus status) {
       String fullPath = status.getFullName(parent);
       boolean hasRemaining = false;
       if (status.isDir()) {
@@ -305,11 +299,11 @@ public class Mover {
           fullPath = fullPath + Path.SEPARATOR;
         }
 
-        hasRemaining = processChildrenList(fullPath);
+        hasRemaining = processPath(fullPath);
         // process snapshots if this is a snapshottable directory
         if (snapshottableDirs.contains(fullPath)) {
           final String dirSnapshot = fullPath + HdfsConstants.DOT_SNAPSHOT_DIR;
-          hasRemaining |= processChildrenList(dirSnapshot);
+          hasRemaining |= processPath(dirSnapshot);
         }
       } else if (!status.isSymlink()) { // file
         try {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/635fdcbe/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
index 0001e3b..a6edd80 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.hdfs.server.mover;
 
-import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 import java.util.ArrayList;
@@ -27,12 +26,10 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
-import com.google.common.base.Joiner;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.ReconfigurationException;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.Path;
@@ -46,7 +43,6 @@ import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.StorageType;
 import org.apache.hadoop.hdfs.protocol.DirectoryListing;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -515,6 +511,42 @@ public class TestStorageMover {
   }
 
   /**
+   * Run Mover with arguments specifying files and directories
+   */
+  @Test
+  public void testMoveSpecificPaths() throws Exception {
+    LOG.info("testMoveSpecificPaths");
+    final Path foo = new Path("/foo");
+    final Path barFile = new Path(foo, "bar");
+    final Path foo2 = new Path("/foo2");
+    final Path bar2File = new Path(foo2, "bar2");
+    Map<Path, BlockStoragePolicy> policyMap = Maps.newHashMap();
+    policyMap.put(foo, COLD);
+    policyMap.put(foo2, WARM);
+    NamespaceScheme nsScheme = new NamespaceScheme(Arrays.asList(foo, foo2),
+        Arrays.asList(barFile, bar2File), BLOCK_SIZE, null, policyMap);
+    ClusterScheme clusterScheme = new ClusterScheme(DEFAULT_CONF,
+        NUM_DATANODES, REPL, genStorageTypes(NUM_DATANODES), null);
+    MigrationTest test = new MigrationTest(clusterScheme, nsScheme);
+    test.setupCluster();
+
+    try {
+      test.prepareNamespace();
+      test.setStoragePolicy();
+
+      Map<URI, List<Path>> map = Mover.Cli.getNameNodePathsToMove(test.conf,
+          "-p", "/foo/bar", "/foo2");
+      int result = Mover.run(map, test.conf);
+      Assert.assertEquals(ExitStatus.SUCCESS.getExitCode(), result);
+
+      Thread.sleep(5000);
+      test.verify(true);
+    } finally {
+      test.shutdownCluster();
+    }
+  }
+
+  /**
    * Move an open file into archival storage
    */
   @Test


Mime
View raw message