hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject svn commit: r718543 - in /hadoop/core/trunk: ./ src/core/org/apache/hadoop/fs/ src/test/org/apache/hadoop/fs/
Date Tue, 18 Nov 2008 10:11:37 GMT
Author: ddas
Date: Tue Nov 18 02:11:36 2008
New Revision: 718543

URL: http://svn.apache.org/viewvc?rev=718543&view=rev
Log:
HADOOP-4185. Adds setVerifyChecksum() method to FileSystem. Contributed by Sharad Agarwal.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/core/org/apache/hadoop/fs/ChecksumFileSystem.java
    hadoop/core/trunk/src/core/org/apache/hadoop/fs/FSInputChecker.java
    hadoop/core/trunk/src/core/org/apache/hadoop/fs/FileSystem.java
    hadoop/core/trunk/src/core/org/apache/hadoop/fs/FilterFileSystem.java
    hadoop/core/trunk/src/core/org/apache/hadoop/fs/FsShell.java
    hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestChecksumFileSystem.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=718543&r1=718542&r2=718543&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue Nov 18 02:11:36 2008
@@ -115,6 +115,9 @@
     the map task and a counter for counting the number of spilled records in both
     map and reduce tasks.
 
+    HADOOP-4185. Adds setVerifyChecksum() method to FileSystem.
+    (Sharad Agarwal via ddas)
+
   OPTIMIZATIONS
 
     HADOOP-3293. Fixes FileInputFormat to do provide locations for splits

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/ChecksumFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/ChecksumFileSystem.java?rev=718543&r1=718542&r2=718543&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/ChecksumFileSystem.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/ChecksumFileSystem.java Tue Nov 18 02:11:36
2008
@@ -39,6 +39,7 @@
 public abstract class ChecksumFileSystem extends FilterFileSystem {
   private static final byte[] CHECKSUM_VERSION = new byte[] {'c', 'r', 'c', 0};
   private int bytesPerChecksum = 512;
+  private boolean verifyChecksum = true;
 
   public static double getApproxChkSumLength(long size) {
     return ChecksumFSOutputSummer.CHKSUM_AS_FRACTION * size;
@@ -54,6 +55,13 @@
       bytesPerChecksum = conf.getInt("io.bytes.per.checksum", 512);
     }
   }
+  
+  /**
+   * Set whether to verify checksum.
+   */
+  public void setVerifyChecksum(boolean verifyChecksum) {
+    this.verifyChecksum = verifyChecksum;
+  }
 
   /** get the raw file system */
   public FileSystem getRawFileSystem() {
@@ -127,14 +135,14 @@
         if (!Arrays.equals(version, CHECKSUM_VERSION))
           throw new IOException("Not a checksum file: "+sumFile);
         this.bytesPerSum = sums.readInt();
-        set(new CRC32(), bytesPerSum, 4);
+        set(fs.verifyChecksum, new CRC32(), bytesPerSum, 4);
       } catch (FileNotFoundException e) {         // quietly ignore
-        set(null, 1, 0);
+        set(fs.verifyChecksum, null, 1, 0);
       } catch (IOException e) {                   // loudly ignore
         LOG.warn("Problem opening checksum file: "+ file + 
                  ".  Ignoring exception: " + 
                  StringUtils.stringifyException(e));
-        set(null, 1, 0);
+        set(fs.verifyChecksum, null, 1, 0);
       }
     }
     
@@ -175,7 +183,7 @@
       if( sums != null ) {
         sums.close();
       }
-      set(null, 1, 0);
+      set(fs.verifyChecksum, null, 1, 0);
     }
     
 

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/FSInputChecker.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/FSInputChecker.java?rev=718543&r1=718542&r2=718543&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/FSInputChecker.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/FSInputChecker.java Tue Nov 18 02:11:36
2008
@@ -69,8 +69,7 @@
   protected FSInputChecker( Path file, int numOfRetries, 
       boolean verifyChecksum, Checksum sum, int chunkSize, int checksumSize ) {
     this(file, numOfRetries);
-    this.verifyChecksum = verifyChecksum;
-    set(sum, chunkSize, checksumSize);
+    set(verifyChecksum, sum, chunkSize, checksumSize);
   }
   
   /** Reads in next checksum chunk data into <code>buf</code> at <code>offset</code>
@@ -393,12 +392,14 @@
   
   /**
    * Set the checksum related parameters
+   * @param verifyChecksum whether to verify checksum
    * @param sum which type of checksum to use
    * @param maxChunkSize maximun chunk size
    * @param checksumSize checksum size
    */
-  final protected synchronized void set(
+  final protected synchronized void set(boolean verifyChecksum,
       Checksum sum, int maxChunkSize, int checksumSize ) {
+    this.verifyChecksum = verifyChecksum;
     this.sum = sum;
     this.buf = new byte[maxChunkSize];
     this.checksum = new byte[checksumSize];

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/FileSystem.java?rev=718543&r1=718542&r2=718543&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/FileSystem.java Tue Nov 18 02:11:36 2008
@@ -1302,6 +1302,15 @@
   public FileChecksum getFileChecksum(Path f) throws IOException {
     return null;
   }
+  
+  /**
+   * Set the verify checksum flag. This is only applicable if the 
+   * corresponding FileSystem supports checksum. By default doesn't do anything.
+   * @param verifyChecksum
+   */
+  public void setVerifyChecksum(boolean verifyChecksum) {
+    //doesn't do anything
+  }
 
   /**
    * Return a list of file status objects that corresponds to the list of paths

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/FilterFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/FilterFileSystem.java?rev=718543&r1=718542&r2=718543&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/FilterFileSystem.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/FilterFileSystem.java Tue Nov 18 02:11:36
2008
@@ -249,6 +249,11 @@
   public FileChecksum getFileChecksum(Path f) throws IOException {
     return fs.getFileChecksum(f);
   }
+  
+  /** {@inheritDoc} */
+  public void setVerifyChecksum(boolean verifyChecksum) {
+    fs.setVerifyChecksum(verifyChecksum);
+  }
 
   @Override
   public Configuration getConf() {

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/FsShell.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/FsShell.java?rev=718543&r1=718542&r2=718543&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/FsShell.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/FsShell.java Tue Nov 18 02:11:36 2008
@@ -44,7 +44,6 @@
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
 
 /** Provide command line access to a FileSystem. */
 public class FsShell extends Configured implements Tool {
@@ -204,9 +203,7 @@
   private FileSystem getSrcFileSystem(Path src, boolean verifyChecksum
       ) throws IOException { 
     FileSystem srcFs = src.getFileSystem(getConf());
-    if (srcFs instanceof DistributedFileSystem) {
-      ((DistributedFileSystem)srcFs).setVerifyChecksum(verifyChecksum);
-    }
+    srcFs.setVerifyChecksum(verifyChecksum);
     return srcFs;
   }
 

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestChecksumFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestChecksumFileSystem.java?rev=718543&r1=718542&r2=718543&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestChecksumFileSystem.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestChecksumFileSystem.java Tue Nov 18
02:11:36 2008
@@ -66,4 +66,43 @@
     inMemFs.delete(testPath, true);
     assertTrue("nothing in the namespace", inMemFs.listStatus(new Path("/")).length == 0);
   }
+  
+  public void testVerifyChecksum() throws Exception {
+    String TEST_ROOT_DIR
+    = System.getProperty("test.build.data","build/test/data/work-dir/localfs");
+    
+    Configuration conf = new Configuration();
+    LocalFileSystem localFs = FileSystem.getLocal(conf);
+    Path testPath = new Path(TEST_ROOT_DIR, "testPath");
+    Path testPath11 = new Path(TEST_ROOT_DIR, "testPath11");
+    FSDataOutputStream fout = localFs.create(testPath);
+    fout.write("testing".getBytes());
+    fout.close();
+    
+    fout = localFs.create(testPath11);
+    fout.write("testing you".getBytes());
+    fout.close();
+    
+    localFs.delete(localFs.getChecksumFile(testPath), true);
+    assertTrue("checksum deleted", !localFs.exists(localFs.getChecksumFile(testPath)));
+    
+    //copying the wrong checksum file
+    FileUtil.copy(localFs, localFs.getChecksumFile(testPath11), localFs, 
+        localFs.getChecksumFile(testPath),false,true,conf);
+    assertTrue("checksum exists", localFs.exists(localFs.getChecksumFile(testPath)));
+    
+    boolean errorRead = false;
+    try {
+      TestLocalFileSystem.readFile(localFs, testPath);
+    }catch(ChecksumException ie) {
+      errorRead = true;
+    }
+    assertTrue("error reading", errorRead);
+    
+    //now setting verify false, the read should succeed
+    localFs.setVerifyChecksum(false);
+    String str = TestLocalFileSystem.readFile(localFs, testPath);
+    assertTrue("read", "testing".equals(str));
+    
+  }
 }



Mime
View raw message