hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ni...@apache.org
Subject svn commit: r565593 - in /lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs: TestDFSUpgradeFromImage.java hadoop-12-dfs-dir.tgz hadoop-12-dfs-dir.txt
Date Tue, 14 Aug 2007 00:51:44 GMT
Author: nigel
Date: Mon Aug 13 17:51:43 2007
New Revision: 565593

URL: http://svn.apache.org/viewvc?view=rev&rev=565593
Log:
HADOOP-1629.  Added a upgrade test for HADOOP-1134. Contributed by Raghu.

Added:
    lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java   (with
props)
    lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/hadoop-12-dfs-dir.tgz   (with props)
    lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/hadoop-12-dfs-dir.txt

Added: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java?view=auto&rev=565593
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java (added)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java Mon Aug
13 17:51:43 2007
@@ -0,0 +1,196 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.dfs;
+
+import junit.framework.TestCase;
+import java.io.*;
+import java.net.InetSocketAddress;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.TreeMap;
+import java.util.zip.CRC32;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Command;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.io.UTF8;
+import org.apache.hadoop.dfs.FSConstants.StartupOption;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * This tests data transfer protocol handling in the Datanode. It sends
+ * various forms of wrong data and verifies that Datanode handles it well.
+ * 
+ * This test uses the following two file from src/test/.../dfs directory :
+ *   1) hadoop-12-dfs-dir.tgz : contains the tar of 
+ *   2) hadoop-12-dfs-dir.txt : checksums that are compared in this test.
+ */
+public class TestDFSUpgradeFromImage extends TestCase {
+  
+  private static final Log LOG = LogFactory.getLog(
+                    "org.apache.hadoop.dfs.TestDFSUpgradeFromImage");
+  
+  int numDataNodes = 4;
+  
+  private static class ReferenceFileInfo {
+    String path;
+    long checksum;
+  }
+  
+  LinkedList<ReferenceFileInfo> refList = new LinkedList<ReferenceFileInfo>();
+  Iterator<ReferenceFileInfo> refIter;
+  
+  boolean printChecksum = false;
+  
+  protected void setUp() throws IOException {
+    String tarFile = System.getProperty("test.cache.data") + 
+                     "/hadoop-12-dfs-dir.tgz";
+    String dataDir = System.getProperty("test.build.data");
+    File dfsDir = new File(dataDir, "dfs");
+    if ( dfsDir.exists() && !FileUtil.fullyDelete(dfsDir) ) {
+      throw new IOException("Could not delete dfs directory '" + dfsDir + "'");
+    }
+    
+    LOG.info("Unpacking the tar file " + tarFile);
+    String[] cmd = { "tar", "-zxf", tarFile, "-C", dataDir };
+    Command.execCommand(cmd);
+    
+    //Now read the reference info
+    
+    BufferedReader reader = new BufferedReader( 
+                        new FileReader(System.getProperty("test.cache.data") +
+                                       "/hadoop-12-dfs-dir.txt"));
+    String line;
+    while ( (line = reader.readLine()) != null ) {
+      
+      line = line.trim();
+      if (line.length() <= 0 || line.startsWith("#")) {
+        continue;
+      }
+      String[] arr = line.split("\\s+\t\\s+");
+      if (arr.length < 1) {
+        continue;
+      }
+      if (arr[0].equals("printChecksums")) {
+        printChecksum = true;
+        break;
+      }
+      if (arr.length < 2) {
+        continue;
+      }
+      ReferenceFileInfo info = new ReferenceFileInfo();
+      info.path = arr[0];
+      info.checksum = Long.parseLong(arr[1]);
+      refList.add(info);
+    }
+    reader.close();
+  }
+
+  private void verifyChecksum(String path, long checksum) throws IOException {
+    if ( refIter == null ) {
+      refIter = refList.iterator();
+    }
+    
+    if ( printChecksum ) {
+      LOG.info("CRC info for reference file : " + path + " \t " + checksum);
+    } else {
+      if ( !refIter.hasNext() ) {
+        throw new IOException("Checking checksum for " + path +
+                              "Not enough elements in the refList");
+      }
+      ReferenceFileInfo info = refIter.next();
+      // The paths are expected to be listed in the same order 
+      // as they are traversed here.
+      assertEquals(info.path, path);
+      assertEquals("Checking checksum for " + path, info.checksum, checksum);
+    }
+  }
+  
+  CRC32 overallChecksum = new CRC32();
+  
+  private void verifyDir(DFSClient client, String dir) 
+                                           throws IOException {
+    
+    DFSFileInfo[] fileArr = client.listPaths(new UTF8(dir));
+    TreeMap<String, Boolean> fileMap = new TreeMap<String, Boolean>();
+    
+    for(DFSFileInfo file : fileArr) {
+      String path = file.getPath().toString();
+      fileMap.put(path, Boolean.valueOf(file.isDir()));
+    }
+    
+    for(Iterator<String> it = fileMap.keySet().iterator(); it.hasNext();) {
+      String path = it.next();
+      boolean isDir = fileMap.get(path);
+      
+      overallChecksum.update(path.getBytes());
+      
+      if ( isDir ) {
+        verifyDir(client, path);
+      } else {
+        // this is not a directory. Checksum the file data.
+        CRC32 fileCRC = new CRC32();
+        FSInputStream in = client.open(new UTF8(path));
+        byte[] buf = new byte[4096];
+        int nRead = 0;
+        while ( (nRead = in.read(buf, 0, buf.length)) > 0 ) {
+          fileCRC.update(buf, 0, nRead);
+        }
+        
+        verifyChecksum(path, fileCRC.getValue());
+      }
+    }
+  }
+  
+  private void verifyFileSystem(DFSClient client) throws IOException {
+  
+    verifyDir(client, "/");
+    
+    verifyChecksum("overallCRC", overallChecksum.getValue());
+    
+    if ( printChecksum ) {
+      throw new IOException("Checksums are written to log as requested. " +
+                            "Throwing this exception to force an error " +
+                            "for this test.");
+    }
+  }
+  
+  public void testUpgradeFromImage() throws IOException {
+    
+    Configuration conf = new Configuration();
+    MiniDFSCluster cluster = new MiniDFSCluster(0, conf, numDataNodes, false,
+                                                true, StartupOption.UPGRADE,
+                                                null);
+    cluster.waitActive();
+    DFSClient dfsClient = new DFSClient(new InetSocketAddress("localhost", 
+                                                  cluster.getNameNodePort()),
+                                        conf);
+    //Safemode will be off only after upgrade is complete. Wait for it.
+    while ( dfsClient.setSafeMode(FSConstants.SafeModeAction.SAFEMODE_GET) ) {
+      LOG.info("Waiting for SafeMode to be OFF.");
+      try {
+        Thread.sleep(1000);
+      } catch (InterruptedException ignored) {}
+    }
+
+    verifyFileSystem(dfsClient);
+  }
+}

Propchange: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java
------------------------------------------------------------------------------
    svn:keywords = Id Revision HeadURL

Added: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/hadoop-12-dfs-dir.tgz
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/hadoop-12-dfs-dir.tgz?view=auto&rev=565593
==============================================================================
Binary file - no diff available.

Propchange: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/hadoop-12-dfs-dir.tgz
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/hadoop-12-dfs-dir.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/hadoop-12-dfs-dir.txt?view=auto&rev=565593
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/hadoop-12-dfs-dir.txt (added)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/hadoop-12-dfs-dir.txt Mon Aug 13 17:51:43
2007
@@ -0,0 +1,69 @@
+#
+# This is a readme for hadoop-12-dir.tgz and hadoop-12-dir.txt.
+#
+# 08/08/2007:
+#
+# See HADOOP-1629 for more info if needed.
+# These two files are used by unit test TestDFSUpgradeFromImage.java 
+# 
+# hadoop-12-dfs-dir.tgz : 
+# ---------------------
+# This file contains the HDFS directory structure for one namenode and 4 datanodes.
+# The structure is setup similar to the structure used in MiniDFSCluster.
+# The directory was created with Hadoo-0.12.x (svn revision 526216).
+#
+# In the test, this directory is unpacked and MiniDFSCluster is run with 
+# "-upgrade" option. The test waits for the upgrade to complete 
+# (leave safe mode) and then all the files are read. The test checks that the
+# directory structure and file checksums exactly match the information
+# in this file.
+#
+# hadoop-12-dfs-dir.txt :
+# ---------------------
+# Along with this description this file contains the expected files and 
+# checksums or the files in the upgraded DFS.
+# 
+# The original DFS directory was created with various types of files and with
+# some recoverable errors (i.e. corrupt or missing .crc files).
+#
+# A similar set of files exist in two different DFS directories. 
+# For e.g. "top-dir-1Mb-12" contains files created with dfs.block.size of 1Mb 
+# and io.bytes.per.checksum of 512.
+#
+# In the future, when Hadoop project no longer supports upgrade from
+# Hadoop-0.12, then a new DFS directory image must be created.
+#
+# To generate checksum info for new files :
+# ---------------------------------------
+# Uncomment the last coment (starts with "printChecksums") and run the 
+# test again. When the test sees this line, it prints the checksum
+# information that should replace the checksum information in 
+# this file. When run in this mode, the test will fail with a descriptive IOException.
+#
+# Next, extract the checksum info from the test log like this:
+#  sed -n 's/.*CRC info for reference file : //p' test-log.txt >> this_file
+# This will append a new list of files and checksums to this file.  Be sure to remove the
existing checksum info.
+#
+# For your reference, the format of the checksum info below is "filename whitespace*\twhitespace*
checksum\n"
+#
+# Uncomment the following line to produce checksum info for a new DFS image.
+#printChecksums
+
+/1kb-multiple-checksum-blocks-64-16 	 191893480
+/top-dir-120000-60/1Mb-file 	 4079112547
+/top-dir-120000-60/4k-file 	 3716287280
+/top-dir-120000-60/5Mb-file 	 2563834633
+/top-dir-120000-60/directory1/500thousand-file 	 3036538664
+/top-dir-120000-60/directory1/file-with-corrupt-crc 	 1984689737
+/top-dir-120000-60/directory1/file-with-no-crc 	 4004594475
+/top-dir-120000-60/directory1/zero1 	 0
+/top-dir-120000-60/zerolen 	 0
+/top-dir-1Mb-512/1Mb-file 	 4079112547
+/top-dir-1Mb-512/4k-file 	 3716287280
+/top-dir-1Mb-512/5Mb-file 	 2563834633
+/top-dir-1Mb-512/directory1/500thousand-file 	 3036538664
+/top-dir-1Mb-512/directory1/file-with-corrupt-crc 	 1984689737
+/top-dir-1Mb-512/directory1/file-with-no-crc 	 4004594475
+/top-dir-1Mb-512/directory1/zero1 	 0
+/top-dir-1Mb-512/zerolen 	 0
+overallCRC 	 1419480698



Mime
View raw message