hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bor...@apache.org
Subject svn commit: r999122 - in /hadoop/hdfs/branches/HDFS-1052: ./ src/java/org/apache/hadoop/hdfs/server/common/ src/java/org/apache/hadoop/hdfs/server/datanode/ src/java/org/apache/hadoop/hdfs/server/namenode/ src/java/org/apache/hadoop/hdfs/server/protoco...
Date Mon, 20 Sep 2010 21:09:40 GMT
Author: boryas
Date: Mon Sep 20 21:09:39 2010
New Revision: 999122

URL: http://svn.apache.org/viewvc?rev=999122&view=rev
Log:
HDFS-1365. HDFS federation: propose ClusterID and BlockPoolID format

Added:
    hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/common/TestStorageInfo.java
Modified:
    hadoop/hdfs/branches/HDFS-1052/CHANGES.txt
    hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
    hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java
    hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
    hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
    hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
    hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
    hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
    hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java
    hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSStartupVersions.java
    hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java
    hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java
    hadoop/hdfs/branches/HDFS-1052/src/test/unit/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java

Modified: hadoop/hdfs/branches/HDFS-1052/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/CHANGES.txt?rev=999122&r1=999121&r2=999122&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/CHANGES.txt (original)
+++ hadoop/hdfs/branches/HDFS-1052/CHANGES.txt Mon Sep 20 21:09:39 2010
@@ -37,6 +37,8 @@ Trunk (unreleased changes)
 
     HDFS-1361. Add -fileStatus operation to NNThroughputBenchmark. (shv)
 
+    HDFS-1365.HDFS federation: propose ClusterID and BlockPoolID format (tanping via boryas)
+
   IMPROVEMENTS
 
     HDFS-1096. fix for prev. commit. (boryas)

Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/Storage.java?rev=999122&r1=999121&r2=999122&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
Mon Sep 20 21:09:39 2010
@@ -665,8 +665,8 @@ public abstract class Storage extends St
     this.storageType = type;
   }
   
-  protected Storage(NodeType type, int nsID, long cT) {
-    super(FSConstants.LAYOUT_VERSION, nsID, cT);
+  protected Storage(NodeType type, int nsID, String cid, String bpid, long cT) {
+    super(FSConstants.LAYOUT_VERSION, nsID, cid, bpid, cT);
     this.storageType = type;
   }
   
@@ -734,20 +734,29 @@ public abstract class Storage extends St
   protected void getFields(Properties props, 
                            StorageDirectory sd 
                            ) throws IOException {
-    String sv, st, sid, sct;
+    String sv, st, sid, scid, sbpid, sct;
     sv = props.getProperty("layoutVersion");
     st = props.getProperty("storageType");
     sid = props.getProperty("namespaceID");
+    scid = props.getProperty("clusterID");
+    sbpid = props.getProperty("blockpoolID");
     sct = props.getProperty("cTime");
-    if (sv == null || st == null || sid == null || sct == null)
+    if (sv == null || st == null || sid == null || scid == null || sbpid == null
+        || sct == null) {
       throw new InconsistentFSStateException(sd.root,
-                                             "file " + STORAGE_FILE_VERSION + " is invalid.");
+        "file " + STORAGE_FILE_VERSION + " is invalid.");
+    }
     int rv = Integer.parseInt(sv);
     NodeType rt = NodeType.valueOf(st);
     int rid = Integer.parseInt(sid);
+    String rcid = scid;
+    String rbpid = sbpid;
     long rct = Long.parseLong(sct);
     if (!storageType.equals(rt) ||
-        !((namespaceID == 0) || (rid == 0) || namespaceID == rid))
+        !((namespaceID == 0) || (rid == 0) || namespaceID == rid) ||
+        !( (clusterID.equals(rcid)) || (clusterID.equals("")) )   ||
+        !( (blockpoolID.equals(rbpid)) || (clusterID.equals("")) )
+        )
       throw new InconsistentFSStateException(sd.root,
                                              "is incompatible with others.");
     if (rv < FSConstants.LAYOUT_VERSION) // future version
@@ -756,6 +765,8 @@ public abstract class Storage extends St
     layoutVersion = rv;
     storageType = rt;
     namespaceID = rid;
+    clusterID = rcid;
+    blockpoolID = rbpid;
     cTime = rct;
   }
   
@@ -772,6 +783,8 @@ public abstract class Storage extends St
     props.setProperty("layoutVersion", String.valueOf(layoutVersion));
     props.setProperty("storageType", storageType.toString());
     props.setProperty("namespaceID", String.valueOf(namespaceID));
+    props.setProperty("clusterID", clusterID);
+    props.setProperty("blockpoolID", blockpoolID);
     props.setProperty("cTime", String.valueOf(cTime));
   }
 
@@ -848,6 +861,8 @@ public abstract class Storage extends St
 
   public static String getRegistrationID(StorageInfo storage) {
     return "NS-" + Integer.toString(storage.getNamespaceID())
+      + "-" + storage.getClusterID()
+      + "-" + storage.getBlockPoolID()
       + "-" + Integer.toString(storage.getLayoutVersion())
       + "-" + Long.toString(storage.getCTime());
   }

Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java?rev=999122&r1=999121&r2=999122&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java
Mon Sep 20 21:09:39 2010
@@ -23,6 +23,7 @@ import java.io.IOException;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
 
 
 /**
@@ -34,14 +35,18 @@ import org.apache.hadoop.io.Writable;
 public class StorageInfo implements Writable {
   public int   layoutVersion;   // layout version of the storage data
   public int   namespaceID;     // id of the file system
+  public String clusterID;      // id of the cluster
+  public String blockpoolID;    // id of the blockpool
   public long  cTime;           // creation time of the file system state
   
   public StorageInfo () {
-    this(0, 0, 0L);
+    this(0, 0, "", "", 0L);
   }
   
-  public StorageInfo(int layoutV, int nsID, long cT) {
+  public StorageInfo(int layoutV, int nsID, String cid, String bpid, long cT) {
     layoutVersion = layoutV;
+    clusterID = cid;
+    blockpoolID = bpid;
     namespaceID = nsID;
     cTime = cT;
   }
@@ -63,13 +68,25 @@ public class StorageInfo implements Writ
   public int    getNamespaceID()  { return namespaceID; }
 
   /**
+   * cluster id of the file system.<p>
+   */
+  public String    getClusterID()  { return clusterID; }
+  
+  /**
+   * blockpool id of the file system.<p>
+   */
+  public String    getBlockPoolID()  { return blockpoolID; }
+  
+  /**
    * Creation time of the file system state.<p>
    * Modified during upgrades.
    */
   public long   getCTime()        { return cTime; }
-
+  
   public void   setStorageInfo(StorageInfo from) {
     layoutVersion = from.layoutVersion;
+    clusterID = from.clusterID;
+    blockpoolID = from.blockpoolID;
     namespaceID = from.namespaceID;
     cTime = from.cTime;
   }
@@ -80,12 +97,16 @@ public class StorageInfo implements Writ
   public void write(DataOutput out) throws IOException {
     out.writeInt(getLayoutVersion());
     out.writeInt(getNamespaceID());
+    WritableUtils.writeString(out, clusterID);
+    WritableUtils.writeString(out, blockpoolID); 
     out.writeLong(getCTime());
   }
 
   public void readFields(DataInput in) throws IOException {
     layoutVersion = in.readInt();
     namespaceID = in.readInt();
+    clusterID = WritableUtils.readString(in);
+    blockpoolID = WritableUtils.readString(in);
     cTime = in.readLong();
   }
 }

Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=999122&r1=999121&r2=999122&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
Mon Sep 20 21:09:39 2010
@@ -348,6 +348,8 @@ public class DataNode extends Configured
         setNewStorageID(dnRegistration);
         dnRegistration.storageInfo.layoutVersion = FSConstants.LAYOUT_VERSION;
         dnRegistration.storageInfo.namespaceID = nsInfo.namespaceID;
+        dnRegistration.storageInfo.clusterID = nsInfo.clusterID;
+        dnRegistration.storageInfo.blockpoolID = nsInfo.blockpoolID;
         // it would have been better to pass storage as a parameter to
         // constructor below - need to augment ReflectionUtils used below.
         conf.set(DFSConfigKeys.DFS_DATANODE_STORAGEID_KEY, dnRegistration.getStorageID());

Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java?rev=999122&r1=999121&r2=999122&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
Mon Sep 20 21:09:39 2010
@@ -66,8 +66,8 @@ public class DataStorage extends Storage
     storageID = "";
   }
   
-  DataStorage(int nsID, long cT, String strgID) {
-    super(NodeType.DATA_NODE, nsID, cT);
+  DataStorage(int nsID, String cID, String bpID, long cT, String strgID) {
+    super(NodeType.DATA_NODE, nsID, cID, bpID, cT);
     this.storageID = strgID;
   }
   
@@ -164,6 +164,8 @@ public class DataStorage extends Storage
     sd.clearDirectory(); // create directory
     this.layoutVersion = FSConstants.LAYOUT_VERSION;
     this.namespaceID = nsInfo.getNamespaceID();
+    this.clusterID = nsInfo.getClusterID();
+    this.blockpoolID = nsInfo.getBlockPoolID();
     this.cTime = 0;
     // store storageID as it currently is
     sd.write();
@@ -238,6 +240,16 @@ public class DataStorage extends Storage
                             "Incompatible namespaceIDs in " + sd.getRoot().getCanonicalPath()
                             + ": namenode namespaceID = " + nsInfo.getNamespaceID() 
                             + "; datanode namespaceID = " + getNamespaceID());
+    if (!getClusterID().equals (nsInfo.getClusterID()))
+      throw new IOException(
+                            "Incompatible clusterIDs in " + sd.getRoot().getCanonicalPath()
+                            + ": namenode clusterID = " + nsInfo.getClusterID() 
+                            + "; datanode clusterID = " + getClusterID());
+    if (!getBlockPoolID().equals(nsInfo.getBlockPoolID()))
+      throw new IOException(
+                            "Incompatible blockpoolIDs in " + sd.getRoot().getCanonicalPath()
+                            + ": namenode blockpoolID = " + nsInfo.getBlockPoolID() 
+                            + "; datanode blockpoolID = " + getBlockPoolID());
     if (this.layoutVersion == FSConstants.LAYOUT_VERSION 
         && this.cTime == nsInfo.getCTime())
       return; // regular startup

Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=999122&r1=999121&r2=999122&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
Mon Sep 20 21:09:39 2010
@@ -29,7 +29,10 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.net.URI;
+import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
+import java.security.NoSuchAlgorithmException;
+import java.security.SecureRandom;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -43,6 +46,7 @@ import java.util.Map;
 import java.util.Properties;
 import java.util.Random;
 import java.util.Set;
+import java.util.UUID;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -73,6 +77,7 @@ import org.apache.hadoop.hdfs.server.pro
 import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.net.DNS;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 
 /**
@@ -159,6 +164,8 @@ public class FSImage extends Storage {
   static private final FsPermission FILE_PERM = new FsPermission((short)0);
   static private final byte[] PATH_SEPARATOR = DFSUtil.string2Bytes(Path.SEPARATOR);
 
+  private static final Random R = new Random();
+  
   /**
    */
   FSImage() {
@@ -1448,6 +1455,51 @@ public class FSImage extends Storage {
     return newID;
   }
 
+  /**
+   * Generate new clusterID.
+   * 
+   * clusterID is a persistent attribute of the cluster.
+   * It is generated when the cluster is created and remains the same
+   * during the life cycle of the cluster.  When a new name node is formated, if 
+   * this is a new cluster, a new clusterID is geneated and stored.  Subsequent 
+   * name node must be given the same ClusterID during its format to be in the 
+   * same cluster.
+   * When a datanode register it receive the clusterID and stick with it.
+   * If at any point, name node or data node tries to join another cluster, it 
+   * will be rejected.
+   * 
+   * @return new clusterID
+   */ 
+  private String newClusterID() {
+    this.clusterID = "cid-" + UUID.randomUUID().toString();
+    return this.clusterID;
+  }
+  
+  /**
+   * Generate new blockpoolID.
+   * 
+   * @return new blockpoolID
+   */ 
+  private String newBlockPoolID() throws UnknownHostException{
+    String ip = "unknownIP";
+    try {
+      ip = DNS.getDefaultIP("default");
+    } catch (UnknownHostException e) {
+      LOG.warn("Could not find ip address of \"default\" inteface.");
+      throw e;
+    }
+    
+    int rand = 0;
+    try {
+      rand = SecureRandom.getInstance("SHA1PRNG").nextInt(Integer.MAX_VALUE);
+    } catch (NoSuchAlgorithmException e) {
+      LOG.warn("Could not use SecureRandom");
+      rand = R.nextInt(Integer.MAX_VALUE);
+    }
+    this.blockpoolID ="BP-" + rand + "-"+ ip + "-" + System.currentTimeMillis();
+    return this.blockpoolID;
+  }
+  
   /** Create new dfs name directory.  Caution: this destroys all files
    * in this filesystem. */
   void format(StorageDirectory sd) throws IOException {
@@ -1465,6 +1517,8 @@ public class FSImage extends Storage {
   public void format() throws IOException {
     this.layoutVersion = FSConstants.LAYOUT_VERSION;
     this.namespaceID = newNamespaceID();
+    this.clusterID = newClusterID();
+    this.blockpoolID = newBlockPoolID();
     this.cTime = 0L;
     this.checkpointTime = now();
     for (Iterator<StorageDirectory> it = 

Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=999122&r1=999121&r2=999122&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Mon Sep 20 21:09:39 2010
@@ -523,6 +523,8 @@ public class FSNamesystem implements FSC
   
   NamespaceInfo getNamespaceInfo() {
     return new NamespaceInfo(dir.fsImage.getNamespaceID(),
+                             dir.fsImage.getClusterID(),
+                             dir.fsImage.getBlockPoolID(),
                              dir.fsImage.getCTime(),
                              getDistributedUpgradeVersion());
   }

Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java?rev=999122&r1=999121&r2=999122&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
Mon Sep 20 21:09:39 2010
@@ -48,8 +48,9 @@ public class NamespaceInfo extends Stora
     buildVersion = null;
   }
   
-  public NamespaceInfo(int nsID, long cT, int duVersion) {
-    super(FSConstants.LAYOUT_VERSION, nsID, cT);
+  public NamespaceInfo(int nsID, String clusterID, String bpID, 
+      long cT, int duVersion) {
+    super(FSConstants.LAYOUT_VERSION, nsID, clusterID, bpID, cT);
     buildVersion = Storage.getBuildVersion();
     this.distributedUpgradeVersion = duVersion;
   }

Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java?rev=999122&r1=999121&r2=999122&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java
Mon Sep 20 21:09:39 2010
@@ -170,6 +170,8 @@ public class TestDFSRollback extends Tes
       UpgradeUtilities.createVersionFile(DATA_NODE, baseDirs,
                                          new StorageInfo(Integer.MIN_VALUE,
                                                          UpgradeUtilities.getCurrentNamespaceID(cluster),
+                                                         UpgradeUtilities.getCurrentClusterID(cluster),
+                                                         UpgradeUtilities.getCurrentBlockPoolID(cluster),
                                                          UpgradeUtilities.getCurrentFsscTime(cluster)));
       startDataNodeShouldFail(StartupOption.ROLLBACK);
       cluster.shutdown();
@@ -185,6 +187,8 @@ public class TestDFSRollback extends Tes
       UpgradeUtilities.createVersionFile(DATA_NODE, baseDirs,
                                          new StorageInfo(UpgradeUtilities.getCurrentLayoutVersion(),
                                                          UpgradeUtilities.getCurrentNamespaceID(cluster),
+                                                         UpgradeUtilities.getCurrentClusterID(cluster),
+                                                         UpgradeUtilities.getCurrentBlockPoolID(cluster),
                                                          Long.MAX_VALUE));
       startDataNodeShouldFail(StartupOption.ROLLBACK);
       cluster.shutdown();
@@ -224,6 +228,8 @@ public class TestDFSRollback extends Tes
       UpgradeUtilities.createVersionFile(NAME_NODE, baseDirs,
                                          new StorageInfo(1,
                                                          UpgradeUtilities.getCurrentNamespaceID(null),
+                                                         UpgradeUtilities.getCurrentClusterID(null),
+                                                         UpgradeUtilities.getCurrentBlockPoolID(null),
                                                          UpgradeUtilities.getCurrentFsscTime(null)));
       startNameNodeShouldFail(StartupOption.UPGRADE);
       UpgradeUtilities.createEmptyDirs(nameNodeDirs);

Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSStartupVersions.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSStartupVersions.java?rev=999122&r1=999121&r2=999122&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSStartupVersions.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSStartupVersions.java
Mon Sep 20 21:09:39 2010
@@ -76,26 +76,28 @@ public class TestDFSStartupVersions exte
     long fsscTimeOld = Long.MIN_VALUE;
     long fsscTimeCur = UpgradeUtilities.getCurrentFsscTime(null);
     long fsscTimeNew = Long.MAX_VALUE;
+    String clusterID = "cid-test";
+    String bpID = "bpid-test";
     
     return new StorageInfo[] {
-      new StorageInfo(layoutVersionOld, namespaceIdCur, fsscTimeOld), // 0
-      new StorageInfo(layoutVersionOld, namespaceIdCur, fsscTimeCur), // 1
-      new StorageInfo(layoutVersionOld, namespaceIdCur, fsscTimeNew), // 2
-      new StorageInfo(layoutVersionOld, namespaceIdOld, fsscTimeOld), // 3
-      new StorageInfo(layoutVersionOld, namespaceIdOld, fsscTimeCur), // 4
-      new StorageInfo(layoutVersionOld, namespaceIdOld, fsscTimeNew), // 5
-      new StorageInfo(layoutVersionCur, namespaceIdCur, fsscTimeOld), // 6
-      new StorageInfo(layoutVersionCur, namespaceIdCur, fsscTimeCur), // 7
-      new StorageInfo(layoutVersionCur, namespaceIdCur, fsscTimeNew), // 8
-      new StorageInfo(layoutVersionCur, namespaceIdOld, fsscTimeOld), // 9
-      new StorageInfo(layoutVersionCur, namespaceIdOld, fsscTimeCur), // 10
-      new StorageInfo(layoutVersionCur, namespaceIdOld, fsscTimeNew), // 11
-      new StorageInfo(layoutVersionNew, namespaceIdCur, fsscTimeOld), // 12
-      new StorageInfo(layoutVersionNew, namespaceIdCur, fsscTimeCur), // 13
-      new StorageInfo(layoutVersionNew, namespaceIdCur, fsscTimeNew), // 14
-      new StorageInfo(layoutVersionNew, namespaceIdOld, fsscTimeOld), // 15
-      new StorageInfo(layoutVersionNew, namespaceIdOld, fsscTimeCur), // 16
-      new StorageInfo(layoutVersionNew, namespaceIdOld, fsscTimeNew), // 17
+      new StorageInfo(layoutVersionOld, namespaceIdCur, clusterID, bpID, fsscTimeOld), //
0
+      new StorageInfo(layoutVersionOld, namespaceIdCur, clusterID, bpID, fsscTimeCur), //
1
+      new StorageInfo(layoutVersionOld, namespaceIdCur, clusterID, bpID, fsscTimeNew), //
2
+      new StorageInfo(layoutVersionOld, namespaceIdOld, clusterID, bpID, fsscTimeOld), //
3
+      new StorageInfo(layoutVersionOld, namespaceIdOld, clusterID, bpID, fsscTimeCur), //
4
+      new StorageInfo(layoutVersionOld, namespaceIdOld, clusterID, bpID, fsscTimeNew), //
5
+      new StorageInfo(layoutVersionCur, namespaceIdCur, clusterID, bpID, fsscTimeOld), //
6
+      new StorageInfo(layoutVersionCur, namespaceIdCur, clusterID, bpID, fsscTimeCur), //
7
+      new StorageInfo(layoutVersionCur, namespaceIdCur, clusterID, bpID, fsscTimeNew), //
8
+      new StorageInfo(layoutVersionCur, namespaceIdOld, clusterID, bpID, fsscTimeOld), //
9
+      new StorageInfo(layoutVersionCur, namespaceIdOld, clusterID, bpID, fsscTimeCur), //
10
+      new StorageInfo(layoutVersionCur, namespaceIdOld, clusterID, bpID, fsscTimeNew), //
11
+      new StorageInfo(layoutVersionNew, namespaceIdCur, clusterID, bpID, fsscTimeOld), //
12
+      new StorageInfo(layoutVersionNew, namespaceIdCur, clusterID, bpID, fsscTimeCur), //
13
+      new StorageInfo(layoutVersionNew, namespaceIdCur, clusterID, bpID, fsscTimeNew), //
14
+      new StorageInfo(layoutVersionNew, namespaceIdOld, clusterID, bpID, fsscTimeOld), //
15
+      new StorageInfo(layoutVersionNew, namespaceIdOld, clusterID, bpID, fsscTimeCur), //
16
+      new StorageInfo(layoutVersionNew, namespaceIdOld, clusterID, bpID, fsscTimeNew), //
17
     };
   }
   
@@ -177,6 +179,8 @@ public class TestDFSStartupVersions exte
     StorageInfo nameNodeVersion = new StorageInfo(
                                                   UpgradeUtilities.getCurrentLayoutVersion(),
                                                   UpgradeUtilities.getCurrentNamespaceID(cluster),
+                                                  UpgradeUtilities.getCurrentClusterID(cluster),
+                                                  UpgradeUtilities.getCurrentBlockPoolID(cluster),
                                                   UpgradeUtilities.getCurrentFsscTime(cluster));
     log("NameNode version info", NAME_NODE, null, nameNodeVersion);
     for (int i = 0; i < versions.length; i++) {

Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java?rev=999122&r1=999121&r2=999122&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java
Mon Sep 20 21:09:39 2010
@@ -175,6 +175,8 @@ public class TestDFSUpgrade extends Test
       UpgradeUtilities.createVersionFile(DATA_NODE, baseDirs,
                                          new StorageInfo(Integer.MIN_VALUE,
                                                          UpgradeUtilities.getCurrentNamespaceID(cluster),
+                                                         UpgradeUtilities.getCurrentClusterID(cluster),
+                                                         UpgradeUtilities.getCurrentBlockPoolID(cluster),
                                                          UpgradeUtilities.getCurrentFsscTime(cluster)));
       startDataNodeShouldFail(StartupOption.REGULAR);
       cluster.shutdown();
@@ -188,6 +190,8 @@ public class TestDFSUpgrade extends Test
       UpgradeUtilities.createVersionFile(DATA_NODE, baseDirs,
                                          new StorageInfo(UpgradeUtilities.getCurrentLayoutVersion(),
                                                          UpgradeUtilities.getCurrentNamespaceID(cluster),
+                                                         UpgradeUtilities.getCurrentClusterID(cluster),
+                                                         UpgradeUtilities.getCurrentBlockPoolID(cluster),
                                                          Long.MAX_VALUE));
       startDataNodeShouldFail(StartupOption.REGULAR);
       cluster.shutdown();
@@ -223,6 +227,8 @@ public class TestDFSUpgrade extends Test
       UpgradeUtilities.createVersionFile(NAME_NODE, baseDirs,
                                          new StorageInfo(Storage.LAST_UPGRADABLE_LAYOUT_VERSION
+ 1,
                                                          UpgradeUtilities.getCurrentNamespaceID(null),
+                                                         UpgradeUtilities.getCurrentClusterID(null),
+                                                         UpgradeUtilities.getCurrentBlockPoolID(null),
                                                          UpgradeUtilities.getCurrentFsscTime(null)));
       startNameNodeShouldFail(StartupOption.UPGRADE);
       UpgradeUtilities.createEmptyDirs(nameNodeDirs);
@@ -232,6 +238,8 @@ public class TestDFSUpgrade extends Test
       UpgradeUtilities.createVersionFile(NAME_NODE, baseDirs,
                                          new StorageInfo(Integer.MIN_VALUE,
                                                          UpgradeUtilities.getCurrentNamespaceID(null),
+                                                         UpgradeUtilities.getCurrentClusterID(null),
+                                                         UpgradeUtilities.getCurrentBlockPoolID(null),
                                                          UpgradeUtilities.getCurrentFsscTime(null)));
       startNameNodeShouldFail(StartupOption.UPGRADE);
       UpgradeUtilities.createEmptyDirs(nameNodeDirs);

Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java?rev=999122&r1=999121&r2=999122&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java
Mon Sep 20 21:09:39 2010
@@ -68,6 +68,10 @@ public class UpgradeUtilities {
   private static long namenodeStorageChecksum;
   // The namespaceId of the namenodeStorage directory
   private static int namenodeStorageNamespaceID;
+  // The clusterId of the namenodeStorage directory
+  private static String namenodeStorageClusterID;
+  // The blockpoolId of the namenodeStorage directory
+  private static String namenodeStorageBlockPoolID;
   // The fsscTime of the namenodeStorage directory
   private static long namenodeStorageFsscTime;
   // The singleton master storage directory for Datanode
@@ -103,6 +107,8 @@ public class UpgradeUtilities {
       NameNode namenode = cluster.getNameNode();
       namenodeStorageNamespaceID = namenode.versionRequest().getNamespaceID();
       namenodeStorageFsscTime = namenode.versionRequest().getCTime();
+      namenodeStorageClusterID = namenode.versionRequest().getClusterID();
+      namenodeStorageBlockPoolID = namenode.versionRequest().getBlockPoolID();
       
       FileSystem fs = FileSystem.get(config);
       Path baseDir = new Path("/TestUpgrade");
@@ -378,6 +384,28 @@ public class UpgradeUtilities {
   }
   
   /**
+   * Return the cluster ID inherent in the currently running
+   * Namenode. 
+   */
+  public static String getCurrentClusterID(MiniDFSCluster cluster) throws IOException {
+    if (cluster != null) {
+      return cluster.getNameNode().versionRequest().getClusterID();
+    }
+    return namenodeStorageClusterID;
+  }
+  
+  /**
+   * Return the blockpool ID inherent in the currently running
+   * Namenode. 
+   */
+  public static String getCurrentBlockPoolID(MiniDFSCluster cluster) throws IOException {
+    if (cluster != null) {
+      return cluster.getNameNode().versionRequest().getBlockPoolID();
+    }
+    return namenodeStorageBlockPoolID;
+  }
+  
+  /**
    * Return the File System State Creation Timestamp (FSSCTime) inherent
    * in the currently running Namenode.  If no Namenode is running,
    * return the FSSCTime of the master Namenode storage directory.

Added: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/common/TestStorageInfo.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/common/TestStorageInfo.java?rev=999122&view=auto
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/common/TestStorageInfo.java
(added)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/common/TestStorageInfo.java
Mon Sep 20 21:09:39 2010
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.common;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import junit.framework.Assert;
+import junit.framework.TestCase;
+
+/**
+ * This is a unit test, which tests {@link Util#stringAsURI(String)}
+ * for IDs being used in HDFS, e.g. ClusterID and BlockPoolID.
+ */
+public class TestStorageInfo extends TestCase {
+
+  /**
+   * Test write() / readFieds() of StroageInfo.  Write StorageInfo into a buffer
+   * then read it back and the result should be the same with the original one.
+   * @throws IOException 
+   */
+  public void testStorageInfo() throws IOException {
+    
+    int nsID = 123;
+    String cid = "cid-test";
+    String bpid = "bpid-test";
+    int layoutV = 234;
+    long cT = 0L;
+    
+    StorageInfo sinfo = new StorageInfo(layoutV, nsID, cid,  bpid, cT);
+    
+    Assert.assertNotNull(sinfo);
+
+    ByteArrayOutputStream bos = new ByteArrayOutputStream();
+    DataOutput output = new DataOutputStream(bos);
+
+    try {
+        // we need to first create an DataOutputStream for sinfo to write into
+        sinfo.write(output);
+        //remember to close the DataOutputStream 
+        //to make sure the data has been written
+        bos.close();
+        
+        // convert ByteArrayInputStream to ByteArrayOutputStream
+        ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
+        DataInputStream dataInputStream = new DataInputStream(bis);
+
+        StorageInfo secondsinfo = new StorageInfo();
+        secondsinfo.readFields(dataInputStream);
+        
+        // compare
+        Assert.assertEquals(sinfo.getClusterID(), secondsinfo.getClusterID());
+        Assert.assertEquals(sinfo.getNamespaceID(), secondsinfo.getNamespaceID());
+        Assert.assertEquals(sinfo.getBlockPoolID(), secondsinfo.getBlockPoolID());
+        Assert.assertEquals(sinfo.getLayoutVersion(), secondsinfo.getLayoutVersion());
+        Assert.assertEquals(sinfo.getCTime(), secondsinfo.getCTime());
+    }catch (IOException e) {
+      e.getMessage();
+    }
+  }
+}
+

Modified: hadoop/hdfs/branches/HDFS-1052/src/test/unit/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/unit/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java?rev=999122&r1=999121&r2=999122&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/unit/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/unit/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
Mon Sep 20 21:09:39 2010
@@ -100,7 +100,8 @@ public class TestBlockRecovery {
     dataDir.mkdirs();
     dirs.add(dataDir);
     DatanodeProtocol namenode = mock(DatanodeProtocol.class);
-    when(namenode.versionRequest()).thenReturn(new NamespaceInfo(1, 1L, 1));
+    when(namenode.versionRequest()).thenReturn(new NamespaceInfo
+        (1, "cid-test", "bpid-test", 1L, 1));
     when(namenode.sendHeartbeat(any(DatanodeRegistration.class), anyLong(), 
         anyLong(), anyLong(), anyInt(), anyInt())).thenReturn(
             new DatanodeCommand[0]);



Mime
View raw message