hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kih...@apache.org
Subject hadoop git commit: HDFS-7795. Show warning if not all favored nodes were chosen by namenode. Contributed by Kihwal Lee.
Date Tue, 17 Feb 2015 19:06:16 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 78a7e8d3a -> db6606223


HDFS-7795. Show warning if not all favored nodes were chosen by namenode. Contributed by Kihwal
Lee.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db660622
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db660622
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db660622

Branch: refs/heads/trunk
Commit: db6606223ca2e17aa7e1b2e2be13c1a19d8e7465
Parents: 78a7e8d
Author: Kihwal Lee <kihwal@apache.org>
Authored: Tue Feb 17 13:05:43 2015 -0600
Committer: Kihwal Lee <kihwal@apache.org>
Committed: Tue Feb 17 13:05:43 2015 -0600

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  3 +++
 .../org/apache/hadoop/hdfs/DFSOutputStream.java | 26 ++++++++++++++------
 2 files changed, 21 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/db660622/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 5e54731..48eb61c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -644,6 +644,9 @@ Release 2.7.0 - UNRELEASED
 
     HDFS-4266. BKJM: Separate write and ack quorum (Rakesh R via umamahesh)
 
+    HDFS-7795. Show warning if not all favored nodes were chosen by namenode
+    (kihwal)
+
   OPTIMIZATIONS
 
     HDFS-7454. Reduce memory footprint for AclEntries in NameNode.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db660622/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index 7d29b3d..85d3410 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -35,6 +35,7 @@ import java.nio.channels.ClosedChannelException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.EnumSet;
+import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
@@ -1443,7 +1444,7 @@ public class DFSOutputStream extends FSOutputSummer
           ExtendedBlock blockCopy = new ExtendedBlock(block);
           blockCopy.setNumBytes(blockSize);
 
-          boolean[] targetPinnings = getPinnings(nodes);
+          boolean[] targetPinnings = getPinnings(nodes, true);
           // send the request
           new Sender(out).writeBlock(blockCopy, nodeStorageTypes[0], accessToken,
               dfsClient.clientName, nodes, nodeStorageTypes, null, bcs, 
@@ -1537,20 +1538,29 @@ public class DFSOutputStream extends FSOutputSummer
       }
     }
 
-    private boolean[] getPinnings(DatanodeInfo[] nodes) {
+    private boolean[] getPinnings(DatanodeInfo[] nodes, boolean shouldLog) {
       if (favoredNodes == null) {
         return null;
       } else {
         boolean[] pinnings = new boolean[nodes.length];
+        HashSet<String> favoredSet =
+            new HashSet<String>(Arrays.asList(favoredNodes));
         for (int i = 0; i < nodes.length; i++) {
-          pinnings[i] = false;
-          for (int j = 0; j < favoredNodes.length; j++) {
-            if (nodes[i].getXferAddrWithHostname().equals(favoredNodes[j])) {
-              pinnings[i] = true;
-              break;
-            }
+          pinnings[i] = favoredSet.remove(nodes[i].getXferAddrWithHostname());
+          if (DFSClient.LOG.isDebugEnabled()) {
+            DFSClient.LOG.debug(nodes[i].getXferAddrWithHostname() +
+                " was chosen by name node (favored=" + pinnings[i] +
+                ").");
           }
         }
+        if (shouldLog && !favoredSet.isEmpty()) {
+          // There is one or more favored nodes that were not allocated.
+          DFSClient.LOG.warn(
+              "These favored nodes were specified but not chosen: " +
+              favoredSet +
+              " Specified favored nodes: " + Arrays.toString(favoredNodes));
+
+        }
         return pinnings;
       }
     }


Mime
View raw message