hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject svn commit: r767331 [2/2] - in /hadoop/core/trunk: ./ bin/ src/docs/src/documentation/content/xdocs/ src/hdfs/org/apache/hadoop/hdfs/server/namenode/ src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/ src/test/org/apache/hadoop/hdfs/tools/ src/t...
Date Tue, 21 Apr 2009 23:39:34 GMT
Added: hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java?rev=767331&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
(added)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
Tue Apr 21 23:39:33 2009
@@ -0,0 +1,371 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Set;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction;
+
+/**
+ * Test function of OfflineImageViewer by:
+ *   * confirming it can correctly process a valid fsimage file and that
+ *     the processing generates a correct representation of the namespace
+ *   * confirming it correctly fails to process an fsimage file with a layout
+ *     version it shouldn't be able to handle
+ *   * confirm it correctly bails on malformed image files, in particular, a
+ *     file that ends suddenly.
+ */
+public class TestOfflineImageViewer extends TestCase {
+
+  // Elements of lines of ls-file output to be compared to FileStatus instance
+  private class LsElements {
+    public String perms;
+    public int replication;
+    public String username;
+    public String groupname;
+    public long filesize;
+    public char dir; // d if dir, - otherwise
+  }
+  
+  // namespace as written to dfs, to be compared with viewer's output
+  final HashMap<String, FileStatus> writtenFiles 
+                                           = new HashMap<String, FileStatus>();
+  
+  
+  private static String ROOT = System.getProperty("test.build.data","/tmp");
+  
+  // Main entry point into testing.  Necessary since we only want to generate
+  // the fsimage file once and use it for multiple tests. 
+  public void testOIV() {
+    File originalFsimage = null;
+    try {
+    originalFsimage = initFsimage();
+    assertNotNull("originalFsImage shouldn't be null", originalFsimage);
+    
+    // Tests:
+    outputOfLSVisitor(originalFsimage);
+    
+    unsupportedFSLayoutVersion(originalFsimage);
+    
+    truncatedFSImage(originalFsimage);
+    
+    } finally {
+      if(originalFsimage != null && originalFsimage.exists())
+        originalFsimage.delete();
+    }
+  }
+
+  // Create a populated namespace for later testing.  Save its contents to a
+  // data structure and store its fsimage location.
+  private File initFsimage() {
+    MiniDFSCluster cluster = null;
+    File orig = null;
+    try {
+      Configuration conf = new Configuration();
+      cluster = new MiniDFSCluster(conf, 4, true, null);
+      FileSystem hdfs = cluster.getFileSystem();
+      
+      int numDirs = 3;
+      int numFilesPerDir = 4;
+      int filesize = 256;
+      
+      // Create a reasonable namespace 
+      for(int i = 0; i < numDirs; i++)  {
+        Path dir = new Path("/dir" + i);
+        hdfs.mkdirs(dir);
+        writtenFiles.put(dir.toString(), pathToFileEntry(hdfs, dir.toString()));
+        for(int j = 0; j < numFilesPerDir; j++) {
+          Path file = new Path(dir, "file" + j);
+          FSDataOutputStream o = hdfs.create(file);
+          o.write(new byte[ filesize++ ]);
+          o.close();
+          
+          writtenFiles.put(file.toString(), pathToFileEntry(hdfs, file.toString()));
+        }
+      }
+
+      // Write results to the fsimage file
+      cluster.getNameNode().setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+      cluster.getNameNode().saveNamespace();
+      
+      // Determine location of fsimage file
+      File [] files = cluster.getNameDirs().toArray(new File[0]);
+      orig =  new File(files[0], "current/fsimage");
+      
+      if(!orig.exists())
+        fail("Didn't generate or can't find fsimage.");
+
+    } catch (IOException e) {
+      fail("Failed trying to generate fsimage file: " + e.getMessage());
+    } finally {
+      if(cluster != null)
+        cluster.shutdown();
+    }
+    return orig;
+  }
+  
+  // Convenience method to generate a file status from file system for 
+  // later comparison
+  private FileStatus pathToFileEntry(FileSystem hdfs, String file) 
+        throws IOException {
+    return hdfs.getFileStatus(new Path(file));
+  }
+
+  // Verify that we can correctly generate an ls-style output for a valid 
+  // fsimage
+  private void outputOfLSVisitor(File originalFsimage) {
+    File testFile = new File(ROOT, "/basicCheck");
+    File outputFile = new File(ROOT, "/basicCheckOutput");
+    
+    try {
+      copyFile(originalFsimage, testFile);
+      
+      ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true);
+      OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false);
+
+      oiv.go();
+      
+      HashMap<String, LsElements> fileOutput = readLsfile(outputFile);
+      
+      compareNamespaces(writtenFiles, fileOutput);
+    } catch (IOException e) {
+      fail("Failed reading valid file: " + e.getMessage());
+    } finally {
+      if(testFile.exists()) testFile.delete();
+      if(outputFile.exists()) outputFile.delete();
+    }
+    System.out.println("Correctly generated ls-style output.");
+  }
+  
+  // Confirm that attempting to read an fsimage file with an unsupported
+  // layout results in an error
+  public void unsupportedFSLayoutVersion(File originalFsimage) {
+    File testFile = new File(ROOT, "/invalidLayoutVersion");
+    File outputFile = new File(ROOT, "invalidLayoutVersionOutput");
+    
+    try {
+      int badVersionNum = -432;
+      changeLayoutVersion(originalFsimage, testFile, badVersionNum);
+      ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true);
+      OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false);
+      
+      try {
+        oiv.go();
+        fail("Shouldn't be able to read invalid laytout version");
+      } catch(IOException e) {
+        if(!e.getMessage().contains(Integer.toString(badVersionNum)))
+          throw e; // wasn't error we were expecting
+        System.out.println("Correctly failed at reading bad image version.");
+      }
+    } catch (IOException e) {
+      fail("Problem testing unsupported layout version: " + e.getMessage());
+    } finally {
+      if(testFile.exists()) testFile.delete();
+      if(outputFile.exists()) outputFile.delete();
+    }
+  }
+  
+  // Verify that image viewer will bail on a file that ends unexpectedly
+  private void truncatedFSImage(File originalFsimage) {
+    File testFile = new File(ROOT, "/truncatedFSImage");
+    File outputFile = new File(ROOT, "/trucnatedFSImageOutput");
+    try {
+      copyPartOfFile(originalFsimage, testFile);
+      assertTrue("Created truncated fsimage", testFile.exists());
+      
+      ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true);
+      OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false);
+
+      try {
+        oiv.go();
+        fail("Managed to process a truncated fsimage file");
+      } catch (EOFException e) {
+        System.out.println("Correctly handled EOF");
+      }
+      
+    } catch (IOException e) {
+      fail("Failed testing truncatedFSImage: " + e.getMessage());
+    } finally {
+      if(testFile.exists()) testFile.delete();
+      if(outputFile.exists()) outputFile.delete();
+    }
+  }
+  
+  // Test that our ls file has all the same compenents of the original namespace
+  private void compareNamespaces(HashMap<String, FileStatus> written,
+      HashMap<String, LsElements> fileOutput) {
+    assertEquals( "Should be the same number of files in both, plus one for root"
+            + " in fileoutput", fileOutput.keySet().size(), 
+                                written.keySet().size() + 1);
+    Set<String> inFile = fileOutput.keySet();
+
+    // For each line in the output file, verify that the namespace had a
+    // filestatus counterpart 
+    for (String path : inFile) {
+      if (path.equals("/")) // root's not included in output from system call
+        continue;
+
+      assertTrue("Path in file (" + path + ") was written to fs", written
+          .containsKey(path));
+      
+      compareFiles(written.get(path), fileOutput.get(path));
+      
+      written.remove(path);
+    }
+
+    assertEquals("No more files were written to fs", 0, written.size());
+  }
+  
+  // Compare two files as listed in the original namespace FileStatus and
+  // the output of the ls file from the image processor
+  private void compareFiles(FileStatus fs, LsElements elements) {
+    assertEquals("directory listed as such",  
+                                        fs.isDir() ? 'd' : '-', elements.dir);
+    assertEquals("perms string equal", 
+                                fs.getPermission().toString(), elements.perms);
+    assertEquals("replication equal", fs.getReplication(), elements.replication);
+    assertEquals("owner equal", fs.getOwner(), elements.username);
+    assertEquals("group equal", fs.getGroup(), elements.groupname);
+    assertEquals("lengths equal", fs.getLen(), elements.filesize);
+  }
+
+  // Read the contents of the file created by the Ls processor
+  private HashMap<String, LsElements> readLsfile(File lsFile) throws IOException {
+    BufferedReader br = new BufferedReader(new FileReader(lsFile));
+    String line = null;
+    HashMap<String, LsElements> fileContents = new HashMap<String, LsElements>();
+    
+    while((line = br.readLine()) != null) 
+      readLsLine(line, fileContents);
+    
+    return fileContents;
+  }
+  
+  // Parse a line from the ls output.  Store permissions, replication, 
+  // username, groupname and filesize in hashmap keyed to the path name
+  private void readLsLine(String line, HashMap<String, LsElements> fileContents) {
+    String elements [] = line.split("\\s+");
+    
+    assertEquals("Not enough elements in ls output", 8, elements.length);
+    
+    LsElements lsLine = new LsElements();
+    
+    lsLine.dir = elements[0].charAt(0);
+    lsLine.perms = elements[0].substring(1);
+    lsLine.replication = elements[1].equals("-") 
+                                             ? 0 : Integer.valueOf(elements[1]);
+    lsLine.username = elements[2];
+    lsLine.groupname = elements[3];
+    lsLine.filesize = Long.valueOf(elements[4]);
+    // skipping date and time 
+    
+    String path = elements[7];
+    
+    // Check that each file in the ls output was listed once
+    assertFalse("LS file had duplicate file entries", 
+        fileContents.containsKey(path));
+    
+    fileContents.put(path, lsLine);
+  }
+  
+  // Copy one fsimage to another, changing the layout version in the process
+  private void changeLayoutVersion(File src, File dest, int newVersion) 
+         throws IOException {
+    DataInputStream in = null; 
+    DataOutputStream out = null; 
+    
+    try {
+      in = new DataInputStream(new FileInputStream(src));
+      out = new DataOutputStream(new FileOutputStream(dest));
+      
+      in.readInt();
+      out.writeInt(newVersion);
+      
+      byte [] b = new byte[1024];
+      while( in.read(b)  > 0 ) {
+        out.write(b);
+      }
+    } finally {
+      if(in != null) in.close();
+      if(out != null) out.close();
+    }
+  }
+  
+  // Only copy part of file into the other.  Used for testing truncated fsimage
+  private void copyPartOfFile(File src, File dest) throws IOException {
+    InputStream in = null;
+    OutputStream out = null;
+    
+    byte [] b = new byte[256];
+    int bytesWritten = 0;
+    int count;
+    int maxBytes = 700;
+    
+    try {
+      in = new FileInputStream(src);
+      out = new FileOutputStream(dest);
+      
+      while( (count = in.read(b))  > 0 && bytesWritten < maxBytes ) {
+        out.write(b);
+        bytesWritten += count;
+      } 
+    } finally {
+      if(in != null) in.close();
+      if(out != null) out.close();
+    }
+  }
+  
+  // Copy one file's contents into the other
+  private void copyFile(File src, File dest) throws IOException {
+    InputStream in = null;
+    OutputStream out = null;
+    
+    try {
+      in = new FileInputStream(src);
+      out = new FileOutputStream(dest);
+
+      byte [] b = new byte[1024];
+      while( in.read(b)  > 0 ) {
+        out.write(b);
+      }
+    } finally {
+      if(in != null) in.close();
+      if(out != null) out.close();
+    }
+  }
+}

Added: hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV18
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV18?rev=767331&view=auto
==============================================================================
Binary file - no diff available.

Propchange: hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV18
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV19
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV19?rev=767331&view=auto
==============================================================================
Binary file - no diff available.

Propchange: hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV19
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream



Mime
View raw message