hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject hadoop git commit: HDFS-5782. Change BlockListAsLongs constructor to take Replica as parameter type instead of concrete classes Block and ReplicaInfo. Contributed by David Powell and Joe Pallas
Date Wed, 14 Jan 2015 21:47:18 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 7fe0f25ad -> 6464a8929


HDFS-5782. Change BlockListAsLongs constructor to take Replica as parameter type instead of
concrete classes Block and ReplicaInfo.  Contributed by David Powell and Joe Pallas


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6464a892
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6464a892
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6464a892

Branch: refs/heads/trunk
Commit: 6464a8929a3623e49155febf2f9817253f9a1de3
Parents: 7fe0f25
Author: Tsz-Wo Nicholas Sze <szetszwo@hortonworks.com>
Authored: Wed Jan 14 13:46:06 2015 -0800
Committer: Tsz-Wo Nicholas Sze <szetszwo@hortonworks.com>
Committed: Wed Jan 14 13:46:06 2015 -0800

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  4 ++
 .../hadoop/hdfs/protocol/BlockListAsLongs.java  | 60 ++++++++++++++++----
 .../server/datanode/SimulatedFSDataset.java     |  4 +-
 .../TestBlockHasMultipleReplicasOnSameDN.java   |  2 +-
 .../server/namenode/NNThroughputBenchmark.java  |  2 +-
 5 files changed, 57 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6464a892/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b879f62d..11d15f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -129,6 +129,10 @@ Trunk (Unreleased)
     HDFS-6609. Use DirectorySnapshottableFeature to represent a snapshottable
     directory. (Jing Zhao via wheat9)
 
+    HDFS-5782. Change BlockListAsLongs constructor to take Replica as parameter
+    type instead of concrete classes Block and ReplicaInfo.  (David Powell
+    and Joe Pallas via szetszwo)
+
   OPTIMIZATIONS
 
   BUG FIXES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6464a892/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
index 8a0b731..4389714 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
@@ -25,7 +25,7 @@ import com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
-import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
+import org.apache.hadoop.hdfs.server.datanode.Replica;
 
 /**
  * This class provides an interface for accessing list of blocks that
@@ -85,8 +85,8 @@ public class BlockListAsLongs implements Iterable<Block> {
    * @param finalized - list of finalized blocks
    * @param uc - list of under construction blocks
    */
-  public BlockListAsLongs(final List<? extends Block> finalized,
-                          final List<ReplicaInfo> uc) {
+  public BlockListAsLongs(final List<? extends Replica> finalized,
+                          final List<? extends Replica> uc) {
     int finalizedSize = finalized == null ? 0 : finalized.size();
     int ucSize = uc == null ? 0 : uc.size();
     int len = HEADER_SIZE
@@ -113,8 +113,34 @@ public class BlockListAsLongs implements Iterable<Block> {
     }
   }
 
+  /**
+   * Create block report from a list of finalized blocks.  Used by
+   * NNThroughputBenchmark.
+   *
+   * @param blocks - list of finalized blocks
+   */
+  public BlockListAsLongs(final List<? extends Block> blocks) {
+    int finalizedSize = blocks == null ? 0 : blocks.size();
+    int len = HEADER_SIZE
+              + (finalizedSize + 1) * LONGS_PER_FINALIZED_BLOCK;
+
+    blockList = new long[len];
+
+    // set the header
+    blockList[0] = finalizedSize;
+    blockList[1] = 0;
+
+    // set finalized blocks
+    for (int i = 0; i < finalizedSize; i++) {
+      setBlock(i, blocks.get(i));
+    }
+
+    // set invalid delimiting block
+    setDelimitingBlock(finalizedSize);
+  }
+
   public BlockListAsLongs() {
-    this(null);
+    this((long[])null);
   }
 
   /**
@@ -279,18 +305,30 @@ public class BlockListAsLongs implements Iterable<Block> {
   /**
    * Set the indexTh block
    * @param index - the index of the block to set
-   * @param b - the block is set to the value of the this block
+   * @param r - the block is set to the value of the this Replica
    */
-  private <T extends Block> void setBlock(final int index, final T b) {
+  private void setBlock(final int index, final Replica r) {
     int pos = index2BlockId(index);
-    blockList[pos] = b.getBlockId();
-    blockList[pos + 1] = b.getNumBytes();
-    blockList[pos + 2] = b.getGenerationStamp();
+    blockList[pos] = r.getBlockId();
+    blockList[pos + 1] = r.getNumBytes();
+    blockList[pos + 2] = r.getGenerationStamp();
     if(index < getNumberOfFinalizedReplicas())
       return;
-    assert ((ReplicaInfo)b).getState() != ReplicaState.FINALIZED :
+    assert r.getState() != ReplicaState.FINALIZED :
       "Must be under-construction replica.";
-    blockList[pos + 3] = ((ReplicaInfo)b).getState().getValue();
+    blockList[pos + 3] = r.getState().getValue();
+  }
+
+  /**
+   * Set the indexTh block
+   * @param index - the index of the block to set
+   * @param b - the block is set to the value of the this Block
+   */
+  private void setBlock(final int index, final Block b) {
+    int pos = index2BlockId(index);
+    blockList[pos] = b.getBlockId();
+    blockList[pos + 1] = b.getNumBytes();
+    blockList[pos + 2] = b.getGenerationStamp();
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6464a892/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index 78eedf9..16e7a20 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -557,12 +557,12 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi>
{
   }
 
   synchronized BlockListAsLongs getBlockReport(String bpid) {
-    final List<Block> blocks = new ArrayList<Block>();
+    final List<Replica> blocks = new ArrayList<Replica>();
     final Map<Block, BInfo> map = blockMap.get(bpid);
     if (map != null) {
       for (BInfo b : map.values()) {
         if (b.isFinalized()) {
-          blocks.add(b.theBlock);
+          blocks.add(b);
         }
       }
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6464a892/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockHasMultipleReplicasOnSameDN.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockHasMultipleReplicasOnSameDN.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockHasMultipleReplicasOnSameDN.java
index e71c0ea..1152c74 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockHasMultipleReplicasOnSameDN.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockHasMultipleReplicasOnSameDN.java
@@ -114,7 +114,7 @@ public class TestBlockHasMultipleReplicasOnSameDN {
     }
 
     for (int i = 0; i < cluster.getStoragesPerDatanode(); ++i) {
-      BlockListAsLongs bll = new BlockListAsLongs(blocks, null);
+      BlockListAsLongs bll = new BlockListAsLongs(blocks);
       FsVolumeSpi v = dn.getFSDataset().getVolumes().get(i);
       DatanodeStorage dns = new DatanodeStorage(v.getStorageID());
       reports[i] = new StorageBlockReport(dns, bll.getBlockListAsLongs());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6464a892/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
index 1fb1c1f..6abe600 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
@@ -977,7 +977,7 @@ public class NNThroughputBenchmark implements Tool {
       // fill remaining slots with blocks that do not exist
       for(int idx = blocks.size()-1; idx >= nrBlocks; idx--)
         blocks.set(idx, new Block(blocks.size() - idx, 0, 0));
-      blockReportList = new BlockListAsLongs(blocks,null).getBlockListAsLongs();
+      blockReportList = new BlockListAsLongs(blocks).getBlockListAsLongs();
     }
 
     long[] getBlockReportList() {


Mime
View raw message