hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ji...@apache.org
Subject hadoop git commit: HDFS-11370. Optimize NamenodeFsck#getReplicaInfo. Contributed Takanobu Asanuma.
Date Wed, 01 Feb 2017 19:21:45 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 59c5f1878 -> b6f290d5b


HDFS-11370. Optimize NamenodeFsck#getReplicaInfo. Contributed Takanobu Asanuma.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6f290d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6f290d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6f290d5

Branch: refs/heads/trunk
Commit: b6f290d5b660ad157c7076767c619d02b3d0f894
Parents: 59c5f18
Author: Jing Zhao <jing9@apache.org>
Authored: Wed Feb 1 11:21:35 2017 -0800
Committer: Jing Zhao <jing9@apache.org>
Committed: Wed Feb 1 11:21:35 2017 -0800

----------------------------------------------------------------------
 .../BlockUnderConstructionFeature.java          | 25 +++++++++++++++++
 .../hdfs/server/namenode/NamenodeFsck.java      | 28 +++++++++++---------
 2 files changed, 41 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6f290d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java
index b935f43..7453184 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java
@@ -24,7 +24,9 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
+import java.util.NoSuchElementException;
 
 import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState.COMPLETE;
 
@@ -110,6 +112,29 @@ public class BlockUnderConstructionFeature {
   }
 
   /**
+   * Note that this iterator doesn't guarantee thread-safe. It depends on
+   * external mechanisms such as the FSNamesystem lock for protection.
+   */
+  public Iterator<DatanodeStorageInfo> getExpectedStorageLocationsIterator() {
+    return new Iterator<DatanodeStorageInfo>() {
+      private int index = 0;
+
+      @Override
+      public boolean hasNext() {
+        return index <  replicas.length;
+      }
+
+      @Override
+      public DatanodeStorageInfo next() {
+        if (!hasNext()) {
+          throw new NoSuchElementException();
+        }
+        return replicas[index++].getExpectedStorageLocation();
+      }
+    };
+  }
+
+  /**
    * @return the index array indicating the block index in each storage. Used
    * only by striped blocks.
    */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6f290d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index 859f541..7e91935 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -589,23 +589,27 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
       return "";
     }
     final boolean isComplete = storedBlock.isComplete();
-    DatanodeStorageInfo[] storages = isComplete ?
-        blockManager.getStorages(storedBlock) :
-        storedBlock.getUnderConstructionFeature().getExpectedStorageLocations();
+    Iterator<DatanodeStorageInfo> storagesItr;
     StringBuilder sb = new StringBuilder(" [");
     final boolean isStriped = storedBlock.isStriped();
     Map<DatanodeStorageInfo, Long> storage2Id = new HashMap<>();
-    if (isStriped && isComplete) {
-      long blockId = storedBlock.getBlockId();
-      Iterable<StorageAndBlockIndex> sis =
-          ((BlockInfoStriped)storedBlock).getStorageAndIndexInfos();
-      for (StorageAndBlockIndex si: sis){
-        storage2Id.put(si.getStorage(), blockId + si.getBlockIndex());
+    if (isComplete) {
+      if (isStriped) {
+        long blockId = storedBlock.getBlockId();
+        Iterable<StorageAndBlockIndex> sis =
+            ((BlockInfoStriped) storedBlock).getStorageAndIndexInfos();
+        for (StorageAndBlockIndex si : sis) {
+          storage2Id.put(si.getStorage(), blockId + si.getBlockIndex());
+        }
       }
+      storagesItr = storedBlock.getStorageInfos();
+    } else {
+      storagesItr = storedBlock.getUnderConstructionFeature()
+          .getExpectedStorageLocationsIterator();
     }
 
-    for (int i = 0; i < storages.length; i++) {
-      DatanodeStorageInfo storage = storages[i];
+    while (storagesItr.hasNext()) {
+      DatanodeStorageInfo storage = storagesItr.next();
       if (isStriped && isComplete) {
         long index = storage2Id.get(storage);
         sb.append("blk_" + index + ":");
@@ -649,7 +653,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
           sb.append("LIVE)");
         }
       }
-      if (i < storages.length - 1) {
+      if (storagesItr.hasNext()) {
         sb.append(", ");
       }
     }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message