hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From whe...@apache.org
Subject hadoop git commit: HDFS-7381. Decouple the management of block id and gen stamps from FSNamesystem. Contributed by Haohui Mai.
Date Tue, 11 Nov 2014 20:42:06 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 f03095984 -> 14a9ff1b9


HDFS-7381. Decouple the management of block id and gen stamps from FSNamesystem. Contributed by Haohui Mai.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/14a9ff1b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/14a9ff1b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/14a9ff1b

Branch: refs/heads/branch-2
Commit: 14a9ff1b9cf948ad667351ab2d3b2bc20ef6127e
Parents: f030959
Author: Haohui Mai <wheat9@apache.org>
Authored: Tue Nov 11 12:41:51 2014 -0800
Committer: Haohui Mai <wheat9@apache.org>
Committed: Tue Nov 11 12:42:02 2014 -0800

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |   3 +
 .../server/blockmanagement/BlockIdManager.java  | 208 ++++++++++++++++++
 .../SequentialBlockIdGenerator.java             |  66 ++++++
 .../hdfs/server/namenode/FSEditLogLoader.java   |   6 +-
 .../hdfs/server/namenode/FSImageFormat.java     |  22 +-
 .../server/namenode/FSImageFormatProtobuf.java  |  23 +-
 .../hdfs/server/namenode/FSNamesystem.java      | 184 ++--------------
 .../namenode/SequentialBlockIdGenerator.java    |  66 ------
 .../blockmanagement/TestSequentialBlockId.java  | 197 +++++++++++++++++
 .../hdfs/server/namenode/TestSaveNamespace.java |  11 +-
 .../server/namenode/TestSequentialBlockId.java  | 209 -------------------
 11 files changed, 523 insertions(+), 472 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/14a9ff1b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index d799fc4..a613a38 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -95,6 +95,9 @@ Release 2.7.0 - UNRELEASED
     HDFS-7365. Remove hdfs.server.blockmanagement.MutableBlockCollection.
     (Li Lu via wheat9)
 
+    HDFS-7381. Decouple the management of block id and gen stamps from
+    FSNamesystem. (wheat9)
+
   OPTIMIZATIONS
 
   BUG FIXES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14a9ff1b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
new file mode 100644
index 0000000..8f547f1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
@@ -0,0 +1,208 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.common.GenerationStamp;
+
+import java.io.IOException;
+
+/**
+ * BlockIdManager allocates the generation stamps and the block ID. The
+ * {@see FSNamesystem} is responsible for persisting the allocations in the
+ * {@see EditLog}.
+ */
+public class BlockIdManager {
+  /**
+   * The global generation stamp for legacy blocks with randomly
+   * generated block IDs.
+   */
+  private final GenerationStamp generationStampV1 = new GenerationStamp();
+  /**
+   * The global generation stamp for this file system.
+   */
+  private final GenerationStamp generationStampV2 = new GenerationStamp();
+  /**
+   * The value of the generation stamp when the first switch to sequential
+   * block IDs was made. Blocks with generation stamps below this value
+   * have randomly allocated block IDs. Blocks with generation stamps above
+   * this value had sequentially allocated block IDs. Read from the fsImage
+   * (or initialized as an offset from the V1 (legacy) generation stamp on
+   * upgrade).
+   */
+  private long generationStampV1Limit;
+  /**
+   * The global block ID space for this file system.
+   */
+  private final SequentialBlockIdGenerator blockIdGenerator;
+
+  public BlockIdManager(BlockManager blockManager) {
+    this.generationStampV1Limit = GenerationStamp.GRANDFATHER_GENERATION_STAMP;
+    this.blockIdGenerator = new SequentialBlockIdGenerator(blockManager);
+  }
+
+  /**
+   * Upgrades the generation stamp for the filesystem
+   * by reserving a sufficient range for all existing blocks.
+   * Should be invoked only during the first upgrade to
+   * sequential block IDs.
+   */
+  public long upgradeGenerationStampToV2() {
+    Preconditions.checkState(generationStampV2.getCurrentValue() ==
+      GenerationStamp.LAST_RESERVED_STAMP);
+    generationStampV2.skipTo(generationStampV1.getCurrentValue() +
+      HdfsConstants.RESERVED_GENERATION_STAMPS_V1);
+
+    generationStampV1Limit = generationStampV2.getCurrentValue();
+    return generationStampV2.getCurrentValue();
+  }
+
+  /**
+   * Sets the generation stamp that delineates random and sequentially
+   * allocated block IDs.
+   *
+   * @param stamp set generation stamp limit to this value
+   */
+  public void setGenerationStampV1Limit(long stamp) {
+    Preconditions.checkState(generationStampV1Limit == GenerationStamp
+      .GRANDFATHER_GENERATION_STAMP);
+    generationStampV1Limit = stamp;
+  }
+
+  /**
+   * Gets the value of the generation stamp that delineates sequential
+   * and random block IDs.
+   */
+  public long getGenerationStampAtblockIdSwitch() {
+    return generationStampV1Limit;
+  }
+
+  @VisibleForTesting
+  SequentialBlockIdGenerator getBlockIdGenerator() {
+    return blockIdGenerator;
+  }
+
+  /**
+   * Sets the maximum allocated block ID for this filesystem. This is
+   * the basis for allocating new block IDs.
+   */
+  public void setLastAllocatedBlockId(long blockId) {
+    blockIdGenerator.skipTo(blockId);
+  }
+
+  /**
+   * Gets the maximum sequentially allocated block ID for this filesystem
+   */
+  public long getLastAllocatedBlockId() {
+    return blockIdGenerator.getCurrentValue();
+  }
+
+  /**
+   * Sets the current generation stamp for legacy blocks
+   */
+  public void setGenerationStampV1(long stamp) {
+    generationStampV1.setCurrentValue(stamp);
+  }
+
+  /**
+   * Gets the current generation stamp for legacy blocks
+   */
+  public long getGenerationStampV1() {
+    return generationStampV1.getCurrentValue();
+  }
+
+  /**
+   * Gets the current generation stamp for this filesystem
+   */
+  public void setGenerationStampV2(long stamp) {
+    generationStampV2.setCurrentValue(stamp);
+  }
+
+  public long getGenerationStampV2() {
+    return generationStampV2.getCurrentValue();
+  }
+
+  /**
+   * Increments, logs and then returns the stamp
+   */
+  public long nextGenerationStamp(boolean legacyBlock) throws IOException {
+    return legacyBlock ? getNextGenerationStampV1() :
+      getNextGenerationStampV2();
+  }
+
+  @VisibleForTesting
+  long getNextGenerationStampV1() throws IOException {
+    long genStampV1 = generationStampV1.nextValue();
+
+    if (genStampV1 >= generationStampV1Limit) {
+      // We ran out of generation stamps for legacy blocks. In practice, it
+      // is extremely unlikely as we reserved 1T v1 generation stamps. The
+      // result is that we can no longer append to the legacy blocks that
+      // were created before the upgrade to sequential block IDs.
+      throw new OutOfV1GenerationStampsException();
+    }
+
+    return genStampV1;
+  }
+
+  @VisibleForTesting
+  long getNextGenerationStampV2() {
+    return generationStampV2.nextValue();
+  }
+
+  public long getGenerationStampV1Limit() {
+    return generationStampV1Limit;
+  }
+
+  /**
+   * Determine whether the block ID was randomly generated (legacy) or
+   * sequentially generated. The generation stamp value is used to
+   * make the distinction.
+   *
+   * @return true if the block ID was randomly generated, false otherwise.
+   */
+  public boolean isLegacyBlock(Block block) {
+    return block.getGenerationStamp() < getGenerationStampV1Limit();
+  }
+
+  /**
+   * Increments, logs and then returns the block ID
+   */
+  public long nextBlockId() {
+    return blockIdGenerator.nextValue();
+  }
+
+  public boolean isGenStampInFuture(Block block) {
+    if (isLegacyBlock(block)) {
+      return block.getGenerationStamp() > getGenerationStampV1();
+    } else {
+      return block.getGenerationStamp() > getGenerationStampV2();
+    }
+  }
+
+  public void clear() {
+    generationStampV1.setCurrentValue(GenerationStamp.LAST_RESERVED_STAMP);
+    generationStampV2.setCurrentValue(GenerationStamp.LAST_RESERVED_STAMP);
+    getBlockIdGenerator().setCurrentValue(SequentialBlockIdGenerator
+      .LAST_RESERVED_BLOCK_ID);
+    setGenerationStampV1Limit(GenerationStamp.GRANDFATHER_GENERATION_STAMP);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14a9ff1b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java
new file mode 100644
index 0000000..eef8857
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.util.SequentialNumber;
+
+/**
+ * Generate the next valid block ID by incrementing the maximum block
+ * ID allocated so far, starting at 2^30+1.
+ *
+ * Block IDs used to be allocated randomly in the past. Hence we may
+ * find some conflicts while stepping through the ID space sequentially.
+ * However given the sparsity of the ID space, conflicts should be rare
+ * and can be skipped over when detected.
+ */
+@InterfaceAudience.Private
+public class SequentialBlockIdGenerator extends SequentialNumber {
+  /**
+   * The last reserved block ID.
+   */
+  public static final long LAST_RESERVED_BLOCK_ID = 1024L * 1024 * 1024;
+
+  private final BlockManager blockManager;
+
+  SequentialBlockIdGenerator(BlockManager blockManagerRef) {
+    super(LAST_RESERVED_BLOCK_ID);
+    this.blockManager = blockManagerRef;
+  }
+
+  @Override // NumberGenerator
+  public long nextValue() {
+    Block b = new Block(super.nextValue());
+
+    // There may be an occasional conflict with randomly generated
+    // block IDs. Skip over the conflicts.
+    while(isValidBlock(b)) {
+      b.setBlockId(super.nextValue());
+    }
+    return b.getBlockId();
+  }
+
+  /**
+   * Returns whether the given block is one pointed-to by a file.
+   */
+  private boolean isValidBlock(Block b) {
+    return (blockManager.getBlockCollection(b) != null);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14a9ff1b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index a8086ba..4603513 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -533,7 +533,7 @@ public class FSEditLogLoader {
     }
     case OP_SET_GENSTAMP_V1: {
       SetGenstampV1Op setGenstampV1Op = (SetGenstampV1Op)op;
-      fsNamesys.setGenerationStampV1(setGenstampV1Op.genStampV1);
+      fsNamesys.getBlockIdManager().setGenerationStampV1(setGenstampV1Op.genStampV1);
       break;
     }
     case OP_SET_PERMISSIONS: {
@@ -726,12 +726,12 @@ public class FSEditLogLoader {
     }
     case OP_SET_GENSTAMP_V2: {
       SetGenstampV2Op setGenstampV2Op = (SetGenstampV2Op) op;
-      fsNamesys.setGenerationStampV2(setGenstampV2Op.genStampV2);
+      fsNamesys.getBlockIdManager().setGenerationStampV2(setGenstampV2Op.genStampV2);
       break;
     }
     case OP_ALLOCATE_BLOCK_ID: {
       AllocateBlockIdOp allocateBlockIdOp = (AllocateBlockIdOp) op;
-      fsNamesys.setLastAllocatedBlockId(allocateBlockIdOp.blockId);
+      fsNamesys.getBlockIdManager().setLastAllocatedBlockId(allocateBlockIdOp.blockId);
       break;
     }
     case OP_ROLLING_UPGRADE_START: {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14a9ff1b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
index 3c58ed9..1f1bf4a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
@@ -342,24 +342,26 @@ public class FSImageFormat {
 
         // read in the last generation stamp for legacy blocks.
         long genstamp = in.readLong();
-        namesystem.setGenerationStampV1(genstamp);
-        
+        namesystem.getBlockIdManager().setGenerationStampV1(genstamp);
+
         if (NameNodeLayoutVersion.supports(
             LayoutVersion.Feature.SEQUENTIAL_BLOCK_ID, imgVersion)) {
           // read the starting generation stamp for sequential block IDs
           genstamp = in.readLong();
-          namesystem.setGenerationStampV2(genstamp);
+          namesystem.getBlockIdManager().setGenerationStampV2(genstamp);
 
           // read the last generation stamp for blocks created after
           // the switch to sequential block IDs.
           long stampAtIdSwitch = in.readLong();
-          namesystem.setGenerationStampV1Limit(stampAtIdSwitch);
+          namesystem.getBlockIdManager().setGenerationStampV1Limit(stampAtIdSwitch);
 
           // read the max sequential block ID.
           long maxSequentialBlockId = in.readLong();
-          namesystem.setLastAllocatedBlockId(maxSequentialBlockId);
+          namesystem.getBlockIdManager().setLastAllocatedBlockId(maxSequentialBlockId);
         } else {
-          long startingGenStamp = namesystem.upgradeGenerationStampToV2();
+
+          long startingGenStamp = namesystem.getBlockIdManager()
+            .upgradeGenerationStampToV2();
           // This is an upgrade.
           LOG.info("Upgrading to sequential block IDs. Generation stamp " +
                    "for new blocks set to " + startingGenStamp);
@@ -1256,10 +1258,10 @@ public class FSImageFormat {
         out.writeInt(sourceNamesystem.unprotectedGetNamespaceInfo()
             .getNamespaceID());
         out.writeLong(numINodes);
-        out.writeLong(sourceNamesystem.getGenerationStampV1());
-        out.writeLong(sourceNamesystem.getGenerationStampV2());
-        out.writeLong(sourceNamesystem.getGenerationStampAtblockIdSwitch());
-        out.writeLong(sourceNamesystem.getLastAllocatedBlockId());
+        out.writeLong(sourceNamesystem.getBlockIdManager().getGenerationStampV1());
+        out.writeLong(sourceNamesystem.getBlockIdManager().getGenerationStampV2());
+        out.writeLong(sourceNamesystem.getBlockIdManager().getGenerationStampAtblockIdSwitch());
+        out.writeLong(sourceNamesystem.getBlockIdManager().getLastAllocatedBlockId());
         out.writeLong(context.getTxId());
         out.writeLong(sourceNamesystem.getLastInodeId());
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14a9ff1b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
index 4387cff..3ee848a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
@@ -46,8 +46,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto;
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
-import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
-import org.apache.hadoop.hdfs.server.common.IncorrectVersionException;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockIdManager;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.CacheManagerSection;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.NameSystemSection;
@@ -293,10 +292,11 @@ public final class FSImageFormatProtobuf {
 
     private void loadNameSystemSection(InputStream in) throws IOException {
       NameSystemSection s = NameSystemSection.parseDelimitedFrom(in);
-      fsn.setGenerationStampV1(s.getGenstampV1());
-      fsn.setGenerationStampV2(s.getGenstampV2());
-      fsn.setGenerationStampV1Limit(s.getGenstampV1Limit());
-      fsn.setLastAllocatedBlockId(s.getLastAllocatedBlockId());
+      BlockIdManager blockIdManager = fsn.getBlockIdManager();
+      blockIdManager.setGenerationStampV1(s.getGenstampV1());
+      blockIdManager.setGenerationStampV2(s.getGenstampV2());
+      blockIdManager.setGenerationStampV1Limit(s.getGenstampV1Limit());
+      blockIdManager.setLastAllocatedBlockId(s.getLastAllocatedBlockId());
       imgTxId = s.getTransactionId();
       if (s.hasRollingUpgradeStartTime()
           && fsn.getFSImage().hasRollbackFSImage()) {
@@ -407,7 +407,7 @@ public final class FSImageFormatProtobuf {
       FileOutputStream fout = new FileOutputStream(file);
       fileChannel = fout.getChannel();
       try {
-        saveInternal(fout, compression, file.getAbsolutePath().toString());
+        saveInternal(fout, compression, file.getAbsolutePath());
       } finally {
         fout.close();
       }
@@ -531,11 +531,12 @@ public final class FSImageFormatProtobuf {
         throws IOException {
       final FSNamesystem fsn = context.getSourceNamesystem();
       OutputStream out = sectionOutputStream;
+      BlockIdManager blockIdManager = fsn.getBlockIdManager();
       NameSystemSection.Builder b = NameSystemSection.newBuilder()
-          .setGenstampV1(fsn.getGenerationStampV1())
-          .setGenstampV1Limit(fsn.getGenerationStampV1Limit())
-          .setGenstampV2(fsn.getGenerationStampV2())
-          .setLastAllocatedBlockId(fsn.getLastAllocatedBlockId())
+          .setGenstampV1(blockIdManager.getGenerationStampV1())
+          .setGenstampV1Limit(blockIdManager.getGenerationStampV1Limit())
+          .setGenstampV2(blockIdManager.getGenerationStampV2())
+          .setLastAllocatedBlockId(blockIdManager.getLastAllocatedBlockId())
           .setTransactionId(context.getTxId());
 
       // We use the non-locked version of getNamespaceInfo here since

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14a9ff1b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 1477eb5..3061459 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -210,6 +210,7 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifie
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager.SecretManagerState;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockIdManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
@@ -217,8 +218,6 @@ import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStatistics;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
-import org.apache.hadoop.hdfs.server.blockmanagement.OutOfV1GenerationStampsException;
-import org.apache.hadoop.hdfs.server.common.GenerationStamp;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
@@ -322,6 +321,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
       }
   };
 
+  private final BlockIdManager blockIdManager;
+
   @VisibleForTesting
   public boolean isAuditEnabled() {
     return !isDefaultAuditLogger || auditLog.isInfoEnabled();
@@ -481,34 +482,6 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
   private final long minBlockSize;         // minimum block size
   private final long maxBlocksPerFile;     // maximum # of blocks per file
 
-  /**
-   * The global generation stamp for legacy blocks with randomly
-   * generated block IDs.
-   */
-  private final GenerationStamp generationStampV1 = new GenerationStamp();
-
-  /**
-   * The global generation stamp for this file system.
-   */
-  private final GenerationStamp generationStampV2 = new GenerationStamp();
-
-  /**
-   * The value of the generation stamp when the first switch to sequential
-   * block IDs was made. Blocks with generation stamps below this value
-   * have randomly allocated block IDs. Blocks with generation stamps above
-   * this value had sequentially allocated block IDs. Read from the fsImage
-   * (or initialized as an offset from the V1 (legacy) generation stamp on
-   * upgrade).
-   */
-  private long generationStampV1Limit =
-      GenerationStamp.GRANDFATHER_GENERATION_STAMP;
-
-  /**
-   * The global block ID space for this file system.
-   */
-  @VisibleForTesting
-  private final SequentialBlockIdGenerator blockIdGenerator;
-
   // precision of access times.
   private final long accessTimePrecision;
 
@@ -640,11 +613,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
   void clear() {
     dir.reset();
     dtSecretManager.reset();
-    generationStampV1.setCurrentValue(GenerationStamp.LAST_RESERVED_STAMP);
-    generationStampV2.setCurrentValue(GenerationStamp.LAST_RESERVED_STAMP);
-    blockIdGenerator.setCurrentValue(
-        SequentialBlockIdGenerator.LAST_RESERVED_BLOCK_ID);
-    generationStampV1Limit = GenerationStamp.GRANDFATHER_GENERATION_STAMP;
+    blockIdManager.clear();
     leaseManager.removeAllLeases();
     inodeId.setCurrentValue(INodeId.LAST_RESERVED_ID);
     snapshotManager.clearSnapshottableDirs();
@@ -794,7 +763,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
 
       this.blockManager = new BlockManager(this, this, conf);
       this.datanodeStatistics = blockManager.getDatanodeManager().getDatanodeStatistics();
-      this.blockIdGenerator = new SequentialBlockIdGenerator(this.blockManager);
+      this.blockIdManager = new BlockIdManager(blockManager);
 
       this.isStoragePolicyEnabled =
           conf.getBoolean(DFS_STORAGE_POLICY_ENABLED_KEY,
@@ -1358,7 +1327,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
    * @throws SafeModeException
    *           Otherwise if NameNode is in SafeMode.
    */
-  private void checkNameNodeSafeMode(String errorMsg)
+  void checkNameNodeSafeMode(String errorMsg)
       throws RetriableException, SafeModeException {
     if (isInSafeMode()) {
       SafeModeException se = new SafeModeException(errorMsg, safeMode);
@@ -4598,7 +4567,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
         return true;
       }
       // start recovery of the last block for this file
-      long blockRecoveryId = nextGenerationStamp(isLegacyBlock(uc));
+      long blockRecoveryId = nextGenerationStamp(blockIdManager.isLegacyBlock(uc));
       lease = reassignLease(lease, src, recoveryLeaseHolder, pendingFile);
       uc.initializeBlockRecovery(blockRecoveryId);
       leaseManager.renewLease(lease);
@@ -6735,91 +6704,6 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
   }
 
   /**
-   * Sets the current generation stamp for legacy blocks
-   */
-  void setGenerationStampV1(long stamp) {
-    generationStampV1.setCurrentValue(stamp);
-  }
-
-  /**
-   * Gets the current generation stamp for legacy blocks
-   */
-  long getGenerationStampV1() {
-    return generationStampV1.getCurrentValue();
-  }
-
-  /**
-   * Gets the current generation stamp for this filesystem
-   */
-  void setGenerationStampV2(long stamp) {
-    generationStampV2.setCurrentValue(stamp);
-  }
-
-  /**
-   * Gets the current generation stamp for this filesystem
-   */
-  long getGenerationStampV2() {
-    return generationStampV2.getCurrentValue();
-  }
-
-  /**
-   * Upgrades the generation stamp for the filesystem
-   * by reserving a sufficient range for all existing blocks.
-   * Should be invoked only during the first upgrade to
-   * sequential block IDs.
-   */
-  long upgradeGenerationStampToV2() {
-    Preconditions.checkState(generationStampV2.getCurrentValue() ==
-        GenerationStamp.LAST_RESERVED_STAMP);
-
-    generationStampV2.skipTo(
-        generationStampV1.getCurrentValue() +
-        HdfsConstants.RESERVED_GENERATION_STAMPS_V1);
-
-    generationStampV1Limit = generationStampV2.getCurrentValue();
-    return generationStampV2.getCurrentValue();
-  }
-
-  /**
-   * Sets the generation stamp that delineates random and sequentially
-   * allocated block IDs.
-   * @param stamp set generation stamp limit to this value
-   */
-  void setGenerationStampV1Limit(long stamp) {
-    Preconditions.checkState(generationStampV1Limit ==
-                             GenerationStamp.GRANDFATHER_GENERATION_STAMP);
-    generationStampV1Limit = stamp;
-  }
-
-  /**
-   * Gets the value of the generation stamp that delineates sequential
-   * and random block IDs.
-   */
-  long getGenerationStampAtblockIdSwitch() {
-    return generationStampV1Limit;
-  }
-
-  @VisibleForTesting
-  SequentialBlockIdGenerator getBlockIdGenerator() {
-    return blockIdGenerator;
-  }
-
-  /**
-   * Sets the maximum allocated block ID for this filesystem. This is
-   * the basis for allocating new block IDs.
-   */
-  void setLastAllocatedBlockId(long blockId) {
-    blockIdGenerator.skipTo(blockId);
-  }
-
-  /**
-   * Gets the maximum sequentially allocated block ID for this filesystem
-   */
-  long getLastAllocatedBlockId() {
-    return blockIdGenerator.getCurrentValue();
-  }
-
-  /**
    * Increments, logs and then returns the stamp
    */
   long nextGenerationStamp(boolean legacyBlock)
@@ -6827,12 +6711,10 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
     assert hasWriteLock();
     checkNameNodeSafeMode("Cannot get next generation stamp");
 
-    long gs;
+    long gs = blockIdManager.nextGenerationStamp(legacyBlock);
     if (legacyBlock) {
-      gs = getNextGenerationStampV1();
       getEditLog().logGenerationStampV1(gs);
     } else {
-      gs = getNextGenerationStampV2();
       getEditLog().logGenerationStampV2(gs);
     }
 
@@ -6840,47 +6722,13 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
     return gs;
   }
 
-  @VisibleForTesting
-  long getNextGenerationStampV1() throws IOException {
-    long genStampV1 = generationStampV1.nextValue();
-
-    if (genStampV1 >= generationStampV1Limit) {
-      // We ran out of generation stamps for legacy blocks. In practice, it
-      // is extremely unlikely as we reserved 1T v1 generation stamps. The
-      // result is that we can no longer append to the legacy blocks that
-      // were created before the upgrade to sequential block IDs.
-      throw new OutOfV1GenerationStampsException();
-    }
-
-    return genStampV1;
-  }
-
-  @VisibleForTesting
-  long getNextGenerationStampV2() {
-    return generationStampV2.nextValue();
-  }
-
-  long getGenerationStampV1Limit() {
-    return generationStampV1Limit;
-  }
-
-  /**
-   * Determine whether the block ID was randomly generated (legacy) or
-   * sequentially generated. The generation stamp value is used to
-   * make the distinction.
-   * @return true if the block ID was randomly generated, false otherwise.
-   */
-  boolean isLegacyBlock(Block block) {
-    return block.getGenerationStamp() < getGenerationStampV1Limit();
-  }
-
   /**
    * Increments, logs and then returns the block ID
    */
   private long nextBlockId() throws IOException {
     assert hasWriteLock();
     checkNameNodeSafeMode("Cannot get next block ID");
-    final long blockId = blockIdGenerator.nextValue();
+    final long blockId = blockIdManager.nextBlockId();
     getEditLog().logAllocateBlockId(blockId);
     // NB: callers sync the log
     return blockId;
@@ -6995,8 +6843,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
       checkUCBlock(block, clientName);
   
       // get a new generation stamp and an access token
-      block.setGenerationStamp(
-          nextGenerationStamp(isLegacyBlock(block.getLocalBlock())));
+      block.setGenerationStamp(nextGenerationStamp(blockIdManager.isLegacyBlock(block.getLocalBlock())));
       locatedBlock = new LocatedBlock(block, new DatanodeInfo[0]);
       blockManager.setBlockToken(locatedBlock, AccessMode.WRITE);
     } finally {
@@ -7868,6 +7715,11 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
   public BlockManager getBlockManager() {
     return blockManager;
   }
+
+  public BlockIdManager getBlockIdManager() {
+    return blockIdManager;
+  }
+
   /** @return the FSDirectory. */
   public FSDirectory getFSDirectory() {
     return dir;
@@ -7935,11 +7787,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
   
   @Override
   public boolean isGenStampInFuture(Block block) {
-    if (isLegacyBlock(block)) {
-      return block.getGenerationStamp() > getGenerationStampV1();
-    } else {
-      return block.getGenerationStamp() > getGenerationStampV2();
-    }
+    return blockIdManager.isGenStampInFuture(block);
   }
 
   @VisibleForTesting

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14a9ff1b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SequentialBlockIdGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SequentialBlockIdGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SequentialBlockIdGenerator.java
deleted file mode 100644
index 650a69b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SequentialBlockIdGenerator.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.namenode;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
-import org.apache.hadoop.util.SequentialNumber;
-
-/**
- * Generate the next valid block ID by incrementing the maximum block
- * ID allocated so far, starting at 2^30+1.
- *
- * Block IDs used to be allocated randomly in the past. Hence we may
- * find some conflicts while stepping through the ID space sequentially.
- * However given the sparsity of the ID space, conflicts should be rare
- * and can be skipped over when detected.
- */
-@InterfaceAudience.Private
-public class SequentialBlockIdGenerator extends SequentialNumber {
-  /**
-   * The last reserved block ID.
-   */
-  public static final long LAST_RESERVED_BLOCK_ID = 1024L * 1024 * 1024;
-
-  private final BlockManager blockManager;
-
-  SequentialBlockIdGenerator(BlockManager blockManagerRef) {
-    super(LAST_RESERVED_BLOCK_ID);
-    this.blockManager = blockManagerRef;
-  }
-
-  @Override // NumberGenerator
-  public long nextValue() {
-    Block b = new Block(super.nextValue());
-
-    // There may be an occasional conflict with randomly generated
-    // block IDs. Skip over the conflicts.
-    while(isValidBlock(b)) {
-      b.setBlockId(super.nextValue());
-    }
-    return b.getBlockId();
-  }
-
-  /**
-   * Returns whether the given block is one pointed-to by a file.
-   */
-  private boolean isValidBlock(Block b) {
-    return (blockManager.getBlockCollection(b) != null);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14a9ff1b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockId.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockId.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockId.java
new file mode 100644
index 0000000..8235294
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockId.java
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
+/**
+ * Tests the sequential block ID generation mechanism and block ID
+ * collision handling.
+ */
+public class TestSequentialBlockId {
+  private static final Log LOG = LogFactory.getLog("TestSequentialBlockId");
+
+  final int BLOCK_SIZE = 1024;
+  final int IO_SIZE = BLOCK_SIZE;
+  final short REPLICATION = 1;
+  final long SEED = 0;
+
+  /**
+   * Test that block IDs are generated sequentially.
+   *
+   * @throws IOException
+   */
+  @Test
+  public void testBlockIdGeneration() throws IOException {
+    Configuration conf = new HdfsConfiguration();
+    conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
+    MiniDFSCluster cluster =
+        new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+
+    try {
+      cluster.waitActive();
+      FileSystem fs = cluster.getFileSystem();
+
+      // Create a file that is 10 blocks long.
+      Path path = new Path("testBlockIdGeneration.dat");
+      DFSTestUtil.createFile(
+          fs, path, IO_SIZE, BLOCK_SIZE * 10, BLOCK_SIZE, REPLICATION, SEED);
+      List<LocatedBlock> blocks = DFSTestUtil.getAllBlocks(fs, path);
+      LOG.info("Block0 id is " + blocks.get(0).getBlock().getBlockId());
+      long nextBlockExpectedId = blocks.get(0).getBlock().getBlockId() + 1;
+
+      // Ensure that the block IDs are sequentially increasing.
+      for (int i = 1; i < blocks.size(); ++i) {
+        long nextBlockId = blocks.get(i).getBlock().getBlockId();
+        LOG.info("Block" + i + " id is " + nextBlockId);
+        assertThat(nextBlockId, is(nextBlockExpectedId));
+        ++nextBlockExpectedId;
+      }
+    } finally {
+      cluster.shutdown();
+    }
+  }
+
+  /**
+   * Test that collisions in the block ID space are handled gracefully.
+   *
+   * @throws IOException
+   */
+  @Test
+  public void testTriggerBlockIdCollision() throws IOException {
+    Configuration conf = new HdfsConfiguration();
+    conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
+    MiniDFSCluster cluster =
+        new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+
+    try {
+      cluster.waitActive();
+      FileSystem fs = cluster.getFileSystem();
+      FSNamesystem fsn = cluster.getNamesystem();
+      final int blockCount = 10;
+
+
+      // Create a file with a few blocks to rev up the global block ID
+      // counter.
+      Path path1 = new Path("testBlockIdCollisionDetection_file1.dat");
+      DFSTestUtil.createFile(
+          fs, path1, IO_SIZE, BLOCK_SIZE * blockCount,
+          BLOCK_SIZE, REPLICATION, SEED);
+      List<LocatedBlock> blocks1 = DFSTestUtil.getAllBlocks(fs, path1);
+
+
+      // Rewind the block ID counter in the name system object. This will result
+      // in block ID collisions when we try to allocate new blocks.
+      SequentialBlockIdGenerator blockIdGenerator = fsn.getBlockIdManager()
+        .getBlockIdGenerator();
+      blockIdGenerator.setCurrentValue(blockIdGenerator.getCurrentValue() - 5);
+
+      // Trigger collisions by creating a new file.
+      Path path2 = new Path("testBlockIdCollisionDetection_file2.dat");
+      DFSTestUtil.createFile(
+          fs, path2, IO_SIZE, BLOCK_SIZE * blockCount,
+          BLOCK_SIZE, REPLICATION, SEED);
+      List<LocatedBlock> blocks2 = DFSTestUtil.getAllBlocks(fs, path2);
+      assertThat(blocks2.size(), is(blockCount));
+
+      // Make sure that file2 block IDs start immediately after file1
+      assertThat(blocks2.get(0).getBlock().getBlockId(),
+                 is(blocks1.get(9).getBlock().getBlockId() + 1));
+
+    } finally {
+      cluster.shutdown();
+    }
+  }
+
+  /**
+   * Test that the block type (legacy or not) can be correctly detected
+   * based on its generation stamp.
+   *
+   * @throws IOException
+   */
+  @Test
+  public void testBlockTypeDetection() throws IOException {
+
+    // Setup a mock object and stub out a few routines to
+    // retrieve the generation stamp counters.
+    BlockIdManager bid = mock(BlockIdManager.class);
+    final long maxGenStampForLegacyBlocks = 10000;
+
+    when(bid.getGenerationStampV1Limit())
+        .thenReturn(maxGenStampForLegacyBlocks);
+
+    Block legacyBlock = spy(new Block());
+    when(legacyBlock.getGenerationStamp())
+        .thenReturn(maxGenStampForLegacyBlocks/2);
+
+    Block newBlock = spy(new Block());
+    when(newBlock.getGenerationStamp())
+        .thenReturn(maxGenStampForLegacyBlocks+1);
+
+    // Make sure that isLegacyBlock() can correctly detect
+    // legacy and new blocks.
+    when(bid.isLegacyBlock(any(Block.class))).thenCallRealMethod();
+    assertThat(bid.isLegacyBlock(legacyBlock), is(true));
+    assertThat(bid.isLegacyBlock(newBlock), is(false));
+  }
+
+  /**
+   * Test that the generation stamp for legacy and new blocks is updated
+   * as expected.
+   *
+   * @throws IOException
+   */
+  @Test
+  public void testGenerationStampUpdate() throws IOException {
+    // Setup a mock object and stub out a few routines to
+    // retrieve the generation stamp counters.
+    BlockIdManager bid = mock(BlockIdManager.class);
+    final long nextGenerationStampV1 = 5000;
+    final long nextGenerationStampV2 = 20000;
+
+    when(bid.getNextGenerationStampV1())
+        .thenReturn(nextGenerationStampV1);
+    when(bid.getNextGenerationStampV2())
+        .thenReturn(nextGenerationStampV2);
+
+    // Make sure that the generation stamp is set correctly for both
+    // kinds of blocks.
+    when(bid.nextGenerationStamp(anyBoolean())).thenCallRealMethod();
+    assertThat(bid.nextGenerationStamp(true), is(nextGenerationStampV1));
+    assertThat(bid.nextGenerationStamp(false), is(nextGenerationStampV2));
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14a9ff1b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
index 26bddc3..5151bf5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockIdManager;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
 import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile;
@@ -544,8 +545,10 @@ public class TestSaveNamespace {
     FSNamesystem spyFsn = spy(fsn);
     final FSNamesystem finalFsn = spyFsn;
     DelayAnswer delayer = new GenericTestUtils.DelayAnswer(LOG);
-    doAnswer(delayer).when(spyFsn).getGenerationStampV2();
-    
+    BlockIdManager bid = spy(spyFsn.getBlockIdManager());
+    Whitebox.setInternalState(finalFsn, "blockIdManager", bid);
+    doAnswer(delayer).when(bid).getGenerationStampV2();
+
     ExecutorService pool = Executors.newFixedThreadPool(2);
     
     try {
@@ -600,9 +603,7 @@ public class TestSaveNamespace {
             NNStorage.getImageFileName(0) + MD5FileUtils.MD5_SUFFIX);
       }      
     } finally {
-      if (fsn != null) {
-        fsn.close();
-      }
+      fsn.close();
     }
   }
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14a9ff1b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSequentialBlockId.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSequentialBlockId.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSequentialBlockId.java
deleted file mode 100644
index e180d75..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSequentialBlockId.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.namenode;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSTestUtil;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.util.DataChecksum;
-import org.junit.Test;
-
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
-
-/**
- * Tests the sequential block ID generation mechanism and block ID
- * collision handling.
- */
-public class TestSequentialBlockId {
-
-  private static final Log LOG = LogFactory.getLog("TestSequentialBlockId");
-
-  private static final DataChecksum DEFAULT_CHECKSUM =
-      DataChecksum.newDataChecksum(DataChecksum.Type.CRC32C, 512);
-
-  final int BLOCK_SIZE = 1024;
-  final int IO_SIZE = BLOCK_SIZE;
-  final short REPLICATION = 1;
-  final long SEED = 0;
-
-  DatanodeID datanode;
-  InetSocketAddress dnAddr;
-
-  /**
-   * Test that block IDs are generated sequentially.
-   *
-   * @throws IOException
-   */
-  @Test
-  public void testBlockIdGeneration() throws IOException {
-    Configuration conf = new HdfsConfiguration();
-    conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
-    MiniDFSCluster cluster =
-        new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
-
-    try {
-      cluster.waitActive();
-      FileSystem fs = cluster.getFileSystem();
-
-      // Create a file that is 10 blocks long.
-      Path path = new Path("testBlockIdGeneration.dat");
-      DFSTestUtil.createFile(
-          fs, path, IO_SIZE, BLOCK_SIZE * 10, BLOCK_SIZE, REPLICATION, SEED);
-      List<LocatedBlock> blocks = DFSTestUtil.getAllBlocks(fs, path);
-      LOG.info("Block0 id is " + blocks.get(0).getBlock().getBlockId());
-      long nextBlockExpectedId = blocks.get(0).getBlock().getBlockId() + 1;
-
-      // Ensure that the block IDs are sequentially increasing.
-      for (int i = 1; i < blocks.size(); ++i) {
-        long nextBlockId = blocks.get(i).getBlock().getBlockId();
-        LOG.info("Block" + i + " id is " + nextBlockId);
-        assertThat(nextBlockId, is(nextBlockExpectedId));
-        ++nextBlockExpectedId;
-      }
-    } finally {
-      cluster.shutdown();
-    }
-  }
-
-  /**
-   * Test that collisions in the block ID space are handled gracefully.
-   *
-   * @throws IOException
-   */
-  @Test
-  public void testTriggerBlockIdCollision() throws IOException {
-    Configuration conf = new HdfsConfiguration();
-    conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
-    MiniDFSCluster cluster =
-        new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
-
-    try {
-      cluster.waitActive();
-      FileSystem fs = cluster.getFileSystem();
-      FSNamesystem fsn = cluster.getNamesystem();
-      final int blockCount = 10;
-
-
-      // Create a file with a few blocks to rev up the global block ID
-      // counter.
-      Path path1 = new Path("testBlockIdCollisionDetection_file1.dat");
-      DFSTestUtil.createFile(
-          fs, path1, IO_SIZE, BLOCK_SIZE * blockCount,
-          BLOCK_SIZE, REPLICATION, SEED);
-      List<LocatedBlock> blocks1 = DFSTestUtil.getAllBlocks(fs, path1);
-
-
-      // Rewind the block ID counter in the name system object. This will result
-      // in block ID collisions when we try to allocate new blocks.
-      SequentialBlockIdGenerator blockIdGenerator = fsn.getBlockIdGenerator();
-      blockIdGenerator.setCurrentValue(blockIdGenerator.getCurrentValue() - 5);
-
-      // Trigger collisions by creating a new file.
-      Path path2 = new Path("testBlockIdCollisionDetection_file2.dat");
-      DFSTestUtil.createFile(
-          fs, path2, IO_SIZE, BLOCK_SIZE * blockCount,
-          BLOCK_SIZE, REPLICATION, SEED);
-      List<LocatedBlock> blocks2 = DFSTestUtil.getAllBlocks(fs, path2);
-      assertThat(blocks2.size(), is(blockCount));
-
-      // Make sure that file2 block IDs start immediately after file1
-      assertThat(blocks2.get(0).getBlock().getBlockId(),
-                 is(blocks1.get(9).getBlock().getBlockId() + 1));
-
-    } finally {
-      cluster.shutdown();
-    }
-  }
-
-  /**
-   * Test that the block type (legacy or not) can be correctly detected
-   * based on its generation stamp.
-   *
-   * @throws IOException
-   */
-  @Test
-  public void testBlockTypeDetection() throws IOException {
-
-    // Setup a mock object and stub out a few routines to
-    // retrieve the generation stamp counters.
-    FSNamesystem fsn = mock(FSNamesystem.class);
-    final long maxGenStampForLegacyBlocks = 10000;
-
-    when(fsn.getGenerationStampV1Limit())
-        .thenReturn(maxGenStampForLegacyBlocks);
-
-    Block legacyBlock = spy(new Block());
-    when(legacyBlock.getGenerationStamp())
-        .thenReturn(maxGenStampForLegacyBlocks/2);
-
-    Block newBlock = spy(new Block());
-    when(newBlock.getGenerationStamp())
-        .thenReturn(maxGenStampForLegacyBlocks+1);
-
-    // Make sure that isLegacyBlock() can correctly detect
-    // legacy and new blocks.
-    when(fsn.isLegacyBlock(any(Block.class))).thenCallRealMethod();
-    assertThat(fsn.isLegacyBlock(legacyBlock), is(true));
-    assertThat(fsn.isLegacyBlock(newBlock), is(false));
-  }
-
-  /**
-   * Test that the generation stamp for legacy and new blocks is updated
-   * as expected.
-   *
-   * @throws IOException
-   */
-  @Test
-  public void testGenerationStampUpdate() throws IOException {
-
-    // Setup a mock object and stub out a few routines to
-    // retrieve the generation stamp counters.
-    FSNamesystem fsn = mock(FSNamesystem.class);
-    FSEditLog editLog = mock(FSEditLog.class);
-    final long nextGenerationStampV1 = 5000;
-    final long nextGenerationStampV2 = 20000;
-
-    when(fsn.getNextGenerationStampV1())
-        .thenReturn(nextGenerationStampV1);
-    when(fsn.getNextGenerationStampV2())
-        .thenReturn(nextGenerationStampV2);
-
-    // Make sure that the generation stamp is set correctly for both
-    // kinds of blocks.
-    when(fsn.nextGenerationStamp(anyBoolean())).thenCallRealMethod();
-    when(fsn.hasWriteLock()).thenReturn(true);
-    when(fsn.getEditLog()).thenReturn(editLog);
-    assertThat(fsn.nextGenerationStamp(true), is(nextGenerationStampV1));
-    assertThat(fsn.nextGenerationStamp(false), is(nextGenerationStampV2));
-  }
-}


Mime
View raw message