hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bor...@apache.org
Subject svn commit: r1075981 - in /hadoop/hdfs/branches/HDFS-1052: ./ src/java/org/apache/hadoop/hdfs/server/namenode/ src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/ src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/
Date Tue, 01 Mar 2011 19:10:49 GMT
Author: boryas
Date: Tue Mar  1 19:10:49 2011
New Revision: 1075981

URL: http://svn.apache.org/viewvc?rev=1075981&view=rev
Log:
HDFS-1662.HDFS federation: fix unit test case, TestCheckpoint and TestDataNodeMXBean

Modified:
    hadoop/hdfs/branches/HDFS-1052/CHANGES.txt
    hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java
    hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
    hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
    hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java
    hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java

Modified: hadoop/hdfs/branches/HDFS-1052/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/CHANGES.txt?rev=1075981&r1=1075980&r2=1075981&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/CHANGES.txt (original)
+++ hadoop/hdfs/branches/HDFS-1052/CHANGES.txt Tue Mar  1 19:10:49 2011
@@ -85,6 +85,9 @@ Trunk (unreleased changes)
 
     HDFS-1670. HDFS Federation: remove dnRegistration from Datanode (boryas)
 
+    HDFS-1662. HDFS federation: fix unit test case, TestCheckpoint 
+    and TestDataNodeMXBean (tanping via boryas)
+
   IMPROVEMENTS
 
     HDFS-1510. Added test-patch.properties required by test-patch.sh (nigel)

Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java?rev=1075981&r1=1075980&r2=1075981&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java
Tue Mar  1 19:10:49 2011
@@ -26,6 +26,7 @@ import org.apache.hadoop.hdfs.server.com
 import org.apache.hadoop.hdfs.server.namenode.FSImage;
 import org.apache.hadoop.io.MD5Hash;
 import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableUtils;
 
 /**
  * A unique signature intended to identify checkpoint transactions.
@@ -37,25 +38,30 @@ public class CheckpointSignature extends
   long editsTime = -1L;
   long checkpointTime = -1L;
   MD5Hash imageDigest = null;
+  String blockpoolID = "";
 
   public CheckpointSignature() {}
 
   CheckpointSignature(FSImage fsImage) {
     super(fsImage);
+    blockpoolID = fsImage.getBlockPoolID();
     editsTime = fsImage.getEditLog().getFsEditTime();
     checkpointTime = fsImage.getCheckpointTime();
     imageDigest = fsImage.imageDigest;
+    checkpointTime = fsImage.getCheckpointTime();
   }
 
   CheckpointSignature(String str) {
     String[] fields = str.split(FIELD_SEPARATOR);
-    assert fields.length == 6 : "Must be 6 fields in CheckpointSignature";
+    assert fields.length == 8 : "Must be 8 fields in CheckpointSignature";
     layoutVersion = Integer.valueOf(fields[0]);
     namespaceID = Integer.valueOf(fields[1]);
     cTime = Long.valueOf(fields[2]);
     editsTime = Long.valueOf(fields[3]);
     checkpointTime = Long.valueOf(fields[4]);
     imageDigest = new MD5Hash(fields[5]);
+    clusterID = fields[6];
+    blockpoolID = fields[7];
   }
 
   /**
@@ -66,28 +72,61 @@ public class CheckpointSignature extends
     return imageDigest;
   }
 
+  /**
+   * Get the cluster id from CheckpointSignature
+   * @return the cluster id
+   */
+  public String getClusterID() {
+    return clusterID;
+  }
+
+  /**
+   * Get the block pool id from CheckpointSignature
+   * @return the block pool id
+   */
+  public String getBlockpoolID() {
+    return blockpoolID;
+  }
+
+  /**
+   * Set the block pool id of CheckpointSignature.
+   * 
+   * @param blockpoolID the new blockpool id
+   */
+  public void setBlockpoolID(String blockpoolID) {
+    this.blockpoolID = blockpoolID;
+  }
+  
   public String toString() {
     return String.valueOf(layoutVersion) + FIELD_SEPARATOR
          + String.valueOf(namespaceID) + FIELD_SEPARATOR
          + String.valueOf(cTime) + FIELD_SEPARATOR
          + String.valueOf(editsTime) + FIELD_SEPARATOR
          + String.valueOf(checkpointTime) + FIELD_SEPARATOR
-         +  imageDigest.toString();
+         + imageDigest.toString() + FIELD_SEPARATOR
+         + clusterID + FIELD_SEPARATOR
+         + blockpoolID ;
   }
 
   void validateStorageInfo(FSImage si) throws IOException {
     if(layoutVersion != si.layoutVersion
         || namespaceID != si.namespaceID || cTime != si.cTime
-        || checkpointTime != si.checkpointTime ||
-        !imageDigest.equals(si.imageDigest)) {
+        || checkpointTime != si.checkpointTime 
+        || !imageDigest.equals(si.imageDigest)
+        || !clusterID.equals(si.clusterID)
+        || !blockpoolID.equals(si.blockpoolID)) {
       // checkpointTime can change when the image is saved - do not compare
       throw new IOException("Inconsistent checkpoint fields.\n"
           + "LV = " + layoutVersion + " namespaceID = " + namespaceID
           + " cTime = " + cTime + "; checkpointTime = " + checkpointTime
           + " ; imageDigest = " + imageDigest
+          + " ; clusterId = " + clusterID
+          + " ; blockpoolId = " + blockpoolID
           + ".\nExpecting respectively: "
           + si.layoutVersion + "; " + si.namespaceID + "; " + si.cTime
-          + "; " + si.checkpointTime + "; " + si.imageDigest);
+          + "; " + si.checkpointTime + "; " + si.imageDigest
+          + "; " + si.clusterID + "; " + si.blockpoolID + "."
+          );
     }
   }
 
@@ -103,6 +142,10 @@ public class CheckpointSignature extends
       (editsTime < o.editsTime) ? -1 : (editsTime > o.editsTime) ? 1 :
       (checkpointTime < o.checkpointTime) ? -1 : 
                   (checkpointTime > o.checkpointTime) ? 1 :
+      (clusterID.compareTo(o.clusterID) < 0) ? -1 : 
+                  (clusterID.compareTo(o.clusterID) > 0) ? 1 :
+      (blockpoolID.compareTo(o.blockpoolID) < 0) ? -1 : 
+                  (blockpoolID.compareTo(o.blockpoolID) > 0) ? 1 :
                     imageDigest.compareTo(o.imageDigest);
   }
 
@@ -116,7 +159,8 @@ public class CheckpointSignature extends
   public int hashCode() {
     return layoutVersion ^ namespaceID ^
             (int)(cTime ^ editsTime ^ checkpointTime) ^
-            imageDigest.hashCode();
+            imageDigest.hashCode() ^ clusterID.hashCode()
+            ^ blockpoolID.hashCode();
   }
 
   /////////////////////////////////////////////////
@@ -124,6 +168,7 @@ public class CheckpointSignature extends
   /////////////////////////////////////////////////
   public void write(DataOutput out) throws IOException {
     super.write(out);
+    WritableUtils.writeString(out, blockpoolID);
     out.writeLong(editsTime);
     out.writeLong(checkpointTime);
     imageDigest.write(out);
@@ -131,6 +176,7 @@ public class CheckpointSignature extends
 
   public void readFields(DataInput in) throws IOException {
     super.readFields(in);
+    blockpoolID = WritableUtils.readString(in);
     editsTime = in.readLong();
     checkpointTime = in.readLong();
     imageDigest = new MD5Hash();

Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1075981&r1=1075980&r2=1075981&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
Tue Mar  1 19:10:49 2011
@@ -97,7 +97,7 @@ public class FSImage extends Storage {
   private static final SimpleDateFormat DATE_FORM =
     new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
 
-  private String blockpoolID = "";   // id of the block pool
+  protected String blockpoolID = "";   // id of the block pool
   
   static final String MESSAGE_DIGEST_PROPERTY = "imageMD5Digest";
   //
@@ -718,6 +718,7 @@ public class FSImage extends Storage {
     realImage.setStorageInfo(ckptImage);
     checkpointTime = ckptImage.checkpointTime;
     fsNamesys.dir.fsImage = realImage;
+    realImage.blockpoolID = ckptImage.blockpoolID;
     // and save it but keep the same checkpointTime
     saveNamespace(false);
   }

Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1075981&r1=1075980&r2=1075981&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
Tue Mar  1 19:10:49 2011
@@ -33,7 +33,9 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSUtil.ErrorSimulator;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.protocol.FSConstants;
 import org.apache.hadoop.hdfs.server.common.HdfsConstants;
 import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
@@ -41,9 +43,6 @@ import org.apache.hadoop.hdfs.server.com
 import org.apache.hadoop.hdfs.server.namenode.FSImage.NameNodeDirType;
 import org.apache.hadoop.hdfs.server.namenode.FSImage.NameNodeFile;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.http.HttpServer;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ipc.RemoteException;
@@ -682,6 +681,8 @@ public class SecondaryNameNode implement
       this.layoutVersion = -1; // to avoid assert in loadFSImage()
       loadFSImage(FSImage.getImageFile(sdName, NameNodeFile.IMAGE));
       loadFSEdits(sdEdits);
+      clusterID = sig.getClusterID();
+      blockpoolID = sig.getBlockpoolID();
       sig.validateStorageInfo(this);
       saveNamespace(false);
     }

Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java?rev=1075981&r1=1075980&r2=1075981&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java
Tue Mar  1 19:10:49 2011
@@ -56,10 +56,10 @@ public class TestDataNodeMXBean {
       // get attribute "HttpPort"
       String httpPort = (String)mbs.getAttribute(mxbeanName, "HttpPort");
       Assert.assertEquals(datanode.getHttpPort(),httpPort);
-      // get attribute "NamenodeAddress"
-      String namenodeAddress = (String)mbs.getAttribute(mxbeanName, 
-          "NamenodeAddress");
-      Assert.assertEquals(datanode.getNamenodeAddresses(),namenodeAddress);
+      // get attribute "NamenodeAddresses"
+      String namenodeAddresses = (String)mbs.getAttribute(mxbeanName, 
+          "NamenodeAddresses");
+      Assert.assertEquals(datanode.getNamenodeAddresses(),namenodeAddresses);
       // get attribute "getVolumeInfo"
       String volumeInfo = (String)mbs.getAttribute(mxbeanName, "VolumeInfo");
       Assert.assertEquals(datanode.getVolumeInfo(),volumeInfo);

Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java?rev=1075981&r1=1075980&r2=1075981&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
Tue Mar  1 19:10:49 2011
@@ -789,4 +789,32 @@ public class TestCheckpoint extends Test
       if(cluster!= null) cluster.shutdown();
     }
   }
+  
+  /* Test case to test CheckpointSignature */
+  @SuppressWarnings("deprecation")
+  public void testCheckpointSignature() throws IOException {
+
+    MiniDFSCluster cluster = null;
+    Configuration conf = new HdfsConfiguration();
+
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes)
+        .format(false).build();
+    NameNode nn = cluster.getNameNode();
+
+    SecondaryNameNode secondary = startSecondaryNameNode(conf);
+    // prepare checkpoint image
+    secondary.doCheckpoint();
+    CheckpointSignature sig = nn.rollEditLog();
+    // manipulate the CheckpointSignature fields
+    sig.setBlockpoolID("somerandomebpid");
+    sig.clusterID = "somerandomcid";
+    try {
+      sig.validateStorageInfo(nn.getFSImage()); // this should fail
+      assertTrue("This test is expected to fail.", false);
+    } catch (Exception ignored) {
+    }
+
+    secondary.shutdown();
+    cluster.shutdown();
+  }
 }



Mime
View raw message