hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1397435 [3/3] - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./ src/main/docs/ src/main/java/org/apache/hadoop/metrics/ src/main/java/org/apache/hadoop/metrics/file/ src/main/java/org/apache/hadoop/metrics/spi/ ...
Date Fri, 12 Oct 2012 04:48:43 GMT
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java?rev=1397435&r1=1397434&r2=1397435&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
Fri Oct 12 04:48:40 2012
@@ -1,460 +1,460 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.util;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.zip.Checksum;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.ChecksumException;
-
-/**
- * This class provides inteface and utilities for processing checksums for
- * DFS data transfers.
- */
-@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
-@InterfaceStability.Evolving
-public class DataChecksum implements Checksum {
-  
-  // Misc constants
-  public static final int HEADER_LEN = 5; /// 1 byte type and 4 byte len
-  
-  // checksum types
-  public static final int CHECKSUM_NULL    = 0;
-  public static final int CHECKSUM_CRC32   = 1;
-  public static final int CHECKSUM_CRC32C  = 2;
-  public static final int CHECKSUM_DEFAULT = 3; 
-  public static final int CHECKSUM_MIXED   = 4;
- 
-  /** The checksum types */
-  public static enum Type {
-    NULL  (CHECKSUM_NULL, 0),
-    CRC32 (CHECKSUM_CRC32, 4),
-    CRC32C(CHECKSUM_CRC32C, 4),
-    DEFAULT(CHECKSUM_DEFAULT, 0), // This cannot be used to create DataChecksum
-    MIXED (CHECKSUM_MIXED, 0); // This cannot be used to create DataChecksum
-
-    public final int id;
-    public final int size;
-    
-    private Type(int id, int size) {
-      this.id = id;
-      this.size = size;
-    }
-
-    /** @return the type corresponding to the id. */
-    public static Type valueOf(int id) {
-      if (id < 0 || id >= values().length) {
-        throw new IllegalArgumentException("id=" + id
-            + " out of range [0, " + values().length + ")");
-      }
-      return values()[id];
-    }
-  }
-
-
-  public static DataChecksum newDataChecksum(Type type, int bytesPerChecksum ) {
-    if ( bytesPerChecksum <= 0 ) {
-      return null;
-    }
-    
-    switch ( type ) {
-    case NULL :
-      return new DataChecksum(type, new ChecksumNull(), bytesPerChecksum );
-    case CRC32 :
-      return new DataChecksum(type, new PureJavaCrc32(), bytesPerChecksum );
-    case CRC32C:
-      return new DataChecksum(type, new PureJavaCrc32C(), bytesPerChecksum);
-    default:
-      return null;  
-    }
-  }
-  
-  /**
-   * Creates a DataChecksum from HEADER_LEN bytes from arr[offset].
-   * @return DataChecksum of the type in the array or null in case of an error.
-   */
-  public static DataChecksum newDataChecksum( byte bytes[], int offset ) {
-    if ( offset < 0 || bytes.length < offset + HEADER_LEN ) {
-      return null;
-    }
-    
-    // like readInt():
-    int bytesPerChecksum = ( (bytes[offset+1] & 0xff) << 24 ) | 
-                           ( (bytes[offset+2] & 0xff) << 16 ) |
-                           ( (bytes[offset+3] & 0xff) << 8 )  |
-                           ( (bytes[offset+4] & 0xff) );
-    return newDataChecksum( Type.valueOf(bytes[0]), bytesPerChecksum );
-  }
-  
-  /**
-   * This constructucts a DataChecksum by reading HEADER_LEN bytes from
-   * input stream <i>in</i>
-   */
-  public static DataChecksum newDataChecksum( DataInputStream in )
-                                 throws IOException {
-    int type = in.readByte();
-    int bpc = in.readInt();
-    DataChecksum summer = newDataChecksum(Type.valueOf(type), bpc );
-    if ( summer == null ) {
-      throw new IOException( "Could not create DataChecksum of type " +
-                             type + " with bytesPerChecksum " + bpc );
-    }
-    return summer;
-  }
-  
-  /**
-   * Writes the checksum header to the output stream <i>out</i>.
-   */
-  public void writeHeader( DataOutputStream out ) 
-                           throws IOException { 
-    out.writeByte( type.id );
-    out.writeInt( bytesPerChecksum );
-  }
-
-  public byte[] getHeader() {
-    byte[] header = new byte[DataChecksum.HEADER_LEN];
-    header[0] = (byte) (type.id & 0xff);
-    // Writing in buffer just like DataOutput.WriteInt()
-    header[1+0] = (byte) ((bytesPerChecksum >>> 24) & 0xff);
-    header[1+1] = (byte) ((bytesPerChecksum >>> 16) & 0xff);
-    header[1+2] = (byte) ((bytesPerChecksum >>> 8) & 0xff);
-    header[1+3] = (byte) (bytesPerChecksum & 0xff);
-    return header;
-  }
-  
-  /**
-   * Writes the current checksum to the stream.
-   * If <i>reset</i> is true, then resets the checksum.
-   * @return number of bytes written. Will be equal to getChecksumSize();
-   */
-   public int writeValue( DataOutputStream out, boolean reset )
-                          throws IOException {
-     if ( type.size <= 0 ) {
-       return 0;
-     }
-
-     if ( type.size == 4 ) {
-       out.writeInt( (int) summer.getValue() );
-     } else {
-       throw new IOException( "Unknown Checksum " + type );
-     }
-     
-     if ( reset ) {
-       reset();
-     }
-     
-     return type.size;
-   }
-   
-   /**
-    * Writes the current checksum to a buffer.
-    * If <i>reset</i> is true, then resets the checksum.
-    * @return number of bytes written. Will be equal to getChecksumSize();
-    */
-    public int writeValue( byte[] buf, int offset, boolean reset )
-                           throws IOException {
-      if ( type.size <= 0 ) {
-        return 0;
-      }
-
-      if ( type.size == 4 ) {
-        int checksum = (int) summer.getValue();
-        buf[offset+0] = (byte) ((checksum >>> 24) & 0xff);
-        buf[offset+1] = (byte) ((checksum >>> 16) & 0xff);
-        buf[offset+2] = (byte) ((checksum >>> 8) & 0xff);
-        buf[offset+3] = (byte) (checksum & 0xff);
-      } else {
-        throw new IOException( "Unknown Checksum " + type );
-      }
-      
-      if ( reset ) {
-        reset();
-      }
-      
-      return type.size;
-    }
-   
-   /**
-    * Compares the checksum located at buf[offset] with the current checksum.
-    * @return true if the checksum matches and false otherwise.
-    */
-   public boolean compare( byte buf[], int offset ) {
-     if ( type.size == 4 ) {
-       int checksum = ( (buf[offset+0] & 0xff) << 24 ) | 
-                      ( (buf[offset+1] & 0xff) << 16 ) |
-                      ( (buf[offset+2] & 0xff) << 8 )  |
-                      ( (buf[offset+3] & 0xff) );
-       return checksum == (int) summer.getValue();
-     }
-     return type.size == 0;
-   }
-   
-  private final Type type;
-  private final Checksum summer;
-  private final int bytesPerChecksum;
-  private int inSum = 0;
-  
-  private DataChecksum( Type type, Checksum checksum, int chunkSize ) {
-    this.type = type;
-    summer = checksum;
-    bytesPerChecksum = chunkSize;
-  }
-  
-  // Accessors
-  public Type getChecksumType() {
-    return type;
-  }
-  public int getChecksumSize() {
-    return type.size;
-  }
-  public int getBytesPerChecksum() {
-    return bytesPerChecksum;
-  }
-  public int getNumBytesInSum() {
-    return inSum;
-  }
-  
-  public static final int SIZE_OF_INTEGER = Integer.SIZE / Byte.SIZE;
-  static public int getChecksumHeaderSize() {
-    return 1 + SIZE_OF_INTEGER; // type byte, bytesPerChecksum int
-  }
-  //Checksum Interface. Just a wrapper around member summer.
-  @Override
-  public long getValue() {
-    return summer.getValue();
-  }
-  @Override
-  public void reset() {
-    summer.reset();
-    inSum = 0;
-  }
-  @Override
-  public void update( byte[] b, int off, int len ) {
-    if ( len > 0 ) {
-      summer.update( b, off, len );
-      inSum += len;
-    }
-  }
-  @Override
-  public void update( int b ) {
-    summer.update( b );
-    inSum += 1;
-  }
-  
-  /**
-   * Verify that the given checksums match the given data.
-   * 
-   * The 'mark' of the ByteBuffer parameters may be modified by this function,.
-   * but the position is maintained.
-   *  
-   * @param data the DirectByteBuffer pointing to the data to verify.
-   * @param checksums the DirectByteBuffer pointing to a series of stored
-   *                  checksums
-   * @param fileName the name of the file being read, for error-reporting
-   * @param basePos the file position to which the start of 'data' corresponds
-   * @throws ChecksumException if the checksums do not match
-   */
-  public void verifyChunkedSums(ByteBuffer data, ByteBuffer checksums,
-      String fileName, long basePos)
-  throws ChecksumException {
-    if (type.size == 0) return;
-    
-    if (data.hasArray() && checksums.hasArray()) {
-      verifyChunkedSums(
-          data.array(), data.arrayOffset() + data.position(), data.remaining(),
-          checksums.array(), checksums.arrayOffset() + checksums.position(),
-          fileName, basePos);
-      return;
-    }
-    if (NativeCrc32.isAvailable()) {
-      NativeCrc32.verifyChunkedSums(bytesPerChecksum, type.id, checksums, data,
-          fileName, basePos);
-      return;
-    }
-    
-    int startDataPos = data.position();
-    data.mark();
-    checksums.mark();
-    try {
-      byte[] buf = new byte[bytesPerChecksum];
-      byte[] sum = new byte[type.size];
-      while (data.remaining() > 0) {
-        int n = Math.min(data.remaining(), bytesPerChecksum);
-        checksums.get(sum);
-        data.get(buf, 0, n);
-        summer.reset();
-        summer.update(buf, 0, n);
-        int calculated = (int)summer.getValue();
-        int stored = (sum[0] << 24 & 0xff000000) |
-          (sum[1] << 16 & 0xff0000) |
-          (sum[2] << 8 & 0xff00) |
-          sum[3] & 0xff;
-        if (calculated != stored) {
-          long errPos = basePos + data.position() - startDataPos - n;
-          throw new ChecksumException(
-              "Checksum error: "+ fileName + " at "+ errPos +
-              " exp: " + stored + " got: " + calculated, errPos);
-        }
-      }
-    } finally {
-      data.reset();
-      checksums.reset();
-    }
-  }
-  
-  /**
-   * Implementation of chunked verification specifically on byte arrays. This
-   * is to avoid the copy when dealing with ByteBuffers that have array backing.
-   */
-  private void verifyChunkedSums(
-      byte[] data, int dataOff, int dataLen,
-      byte[] checksums, int checksumsOff, String fileName,
-      long basePos) throws ChecksumException {
-    
-    int remaining = dataLen;
-    int dataPos = 0;
-    while (remaining > 0) {
-      int n = Math.min(remaining, bytesPerChecksum);
-      
-      summer.reset();
-      summer.update(data, dataOff + dataPos, n);
-      dataPos += n;
-      remaining -= n;
-      
-      int calculated = (int)summer.getValue();
-      int stored = (checksums[checksumsOff] << 24 & 0xff000000) |
-        (checksums[checksumsOff + 1] << 16 & 0xff0000) |
-        (checksums[checksumsOff + 2] << 8 & 0xff00) |
-        checksums[checksumsOff + 3] & 0xff;
-      checksumsOff += 4;
-      if (calculated != stored) {
-        long errPos = basePos + dataPos - n;
-        throw new ChecksumException(
-            "Checksum error: "+ fileName + " at "+ errPos +
-            " exp: " + stored + " got: " + calculated, errPos);
-      }
-    }
-  }
-
-  /**
-   * Calculate checksums for the given data.
-   * 
-   * The 'mark' of the ByteBuffer parameters may be modified by this function,
-   * but the position is maintained.
-   * 
-   * @param data the DirectByteBuffer pointing to the data to checksum.
-   * @param checksums the DirectByteBuffer into which checksums will be
-   *                  stored. Enough space must be available in this
-   *                  buffer to put the checksums.
-   */
-  public void calculateChunkedSums(ByteBuffer data, ByteBuffer checksums) {
-    if (type.size == 0) return;
-    
-    if (data.hasArray() && checksums.hasArray()) {
-      calculateChunkedSums(data.array(), data.arrayOffset() + data.position(), data.remaining(),
-          checksums.array(), checksums.arrayOffset() + checksums.position());
-      return;
-    }
-    
-    data.mark();
-    checksums.mark();
-    try {
-      byte[] buf = new byte[bytesPerChecksum];
-      while (data.remaining() > 0) {
-        int n = Math.min(data.remaining(), bytesPerChecksum);
-        data.get(buf, 0, n);
-        summer.reset();
-        summer.update(buf, 0, n);
-        checksums.putInt((int)summer.getValue());
-      }
-    } finally {
-      data.reset();
-      checksums.reset();
-    }
-  }
-
-  /**
-   * Implementation of chunked calculation specifically on byte arrays. This
-   * is to avoid the copy when dealing with ByteBuffers that have array backing.
-   */
-  private void calculateChunkedSums(
-      byte[] data, int dataOffset, int dataLength,
-      byte[] sums, int sumsOffset) {
-
-    int remaining = dataLength;
-    while (remaining > 0) {
-      int n = Math.min(remaining, bytesPerChecksum);
-      summer.reset();
-      summer.update(data, dataOffset, n);
-      dataOffset += n;
-      remaining -= n;
-      long calculated = summer.getValue();
-      sums[sumsOffset++] = (byte) (calculated >> 24);
-      sums[sumsOffset++] = (byte) (calculated >> 16);
-      sums[sumsOffset++] = (byte) (calculated >> 8);
-      sums[sumsOffset++] = (byte) (calculated);
-    }
-  }
-
-  @Override
-  public boolean equals(Object other) {
-    if (!(other instanceof DataChecksum)) {
-      return false;
-    }
-    DataChecksum o = (DataChecksum)other;
-    return o.bytesPerChecksum == this.bytesPerChecksum &&
-      o.type == this.type;
-  }
-  
-  @Override
-  public int hashCode() {
-    return (this.type.id + 31) * this.bytesPerChecksum;
-  }
-  
-  @Override
-  public String toString() {
-    return "DataChecksum(type=" + type +
-      ", chunkSize=" + bytesPerChecksum + ")";
-  }
-  
-  /**
-   * This just provides a dummy implimentation for Checksum class
-   * This is used when there is no checksum available or required for 
-   * data
-   */
-  static class ChecksumNull implements Checksum {
-    
-    public ChecksumNull() {}
-    
-    //Dummy interface
-    @Override
-    public long getValue() { return 0; }
-    @Override
-    public void reset() {}
-    @Override
-    public void update(byte[] b, int off, int len) {}
-    @Override
-    public void update(int b) {}
-  };
-}
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.zip.Checksum;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.ChecksumException;
+
+/**
+ * This class provides inteface and utilities for processing checksums for
+ * DFS data transfers.
+ */
+@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
+@InterfaceStability.Evolving
+public class DataChecksum implements Checksum {
+  
+  // Misc constants
+  public static final int HEADER_LEN = 5; /// 1 byte type and 4 byte len
+  
+  // checksum types
+  public static final int CHECKSUM_NULL    = 0;
+  public static final int CHECKSUM_CRC32   = 1;
+  public static final int CHECKSUM_CRC32C  = 2;
+  public static final int CHECKSUM_DEFAULT = 3; 
+  public static final int CHECKSUM_MIXED   = 4;
+ 
+  /** The checksum types */
+  public static enum Type {
+    NULL  (CHECKSUM_NULL, 0),
+    CRC32 (CHECKSUM_CRC32, 4),
+    CRC32C(CHECKSUM_CRC32C, 4),
+    DEFAULT(CHECKSUM_DEFAULT, 0), // This cannot be used to create DataChecksum
+    MIXED (CHECKSUM_MIXED, 0); // This cannot be used to create DataChecksum
+
+    public final int id;
+    public final int size;
+    
+    private Type(int id, int size) {
+      this.id = id;
+      this.size = size;
+    }
+
+    /** @return the type corresponding to the id. */
+    public static Type valueOf(int id) {
+      if (id < 0 || id >= values().length) {
+        throw new IllegalArgumentException("id=" + id
+            + " out of range [0, " + values().length + ")");
+      }
+      return values()[id];
+    }
+  }
+
+
+  public static DataChecksum newDataChecksum(Type type, int bytesPerChecksum ) {
+    if ( bytesPerChecksum <= 0 ) {
+      return null;
+    }
+    
+    switch ( type ) {
+    case NULL :
+      return new DataChecksum(type, new ChecksumNull(), bytesPerChecksum );
+    case CRC32 :
+      return new DataChecksum(type, new PureJavaCrc32(), bytesPerChecksum );
+    case CRC32C:
+      return new DataChecksum(type, new PureJavaCrc32C(), bytesPerChecksum);
+    default:
+      return null;  
+    }
+  }
+  
+  /**
+   * Creates a DataChecksum from HEADER_LEN bytes from arr[offset].
+   * @return DataChecksum of the type in the array or null in case of an error.
+   */
+  public static DataChecksum newDataChecksum( byte bytes[], int offset ) {
+    if ( offset < 0 || bytes.length < offset + HEADER_LEN ) {
+      return null;
+    }
+    
+    // like readInt():
+    int bytesPerChecksum = ( (bytes[offset+1] & 0xff) << 24 ) | 
+                           ( (bytes[offset+2] & 0xff) << 16 ) |
+                           ( (bytes[offset+3] & 0xff) << 8 )  |
+                           ( (bytes[offset+4] & 0xff) );
+    return newDataChecksum( Type.valueOf(bytes[0]), bytesPerChecksum );
+  }
+  
+  /**
+   * This constructucts a DataChecksum by reading HEADER_LEN bytes from
+   * input stream <i>in</i>
+   */
+  public static DataChecksum newDataChecksum( DataInputStream in )
+                                 throws IOException {
+    int type = in.readByte();
+    int bpc = in.readInt();
+    DataChecksum summer = newDataChecksum(Type.valueOf(type), bpc );
+    if ( summer == null ) {
+      throw new IOException( "Could not create DataChecksum of type " +
+                             type + " with bytesPerChecksum " + bpc );
+    }
+    return summer;
+  }
+  
+  /**
+   * Writes the checksum header to the output stream <i>out</i>.
+   */
+  public void writeHeader( DataOutputStream out ) 
+                           throws IOException { 
+    out.writeByte( type.id );
+    out.writeInt( bytesPerChecksum );
+  }
+
+  public byte[] getHeader() {
+    byte[] header = new byte[DataChecksum.HEADER_LEN];
+    header[0] = (byte) (type.id & 0xff);
+    // Writing in buffer just like DataOutput.WriteInt()
+    header[1+0] = (byte) ((bytesPerChecksum >>> 24) & 0xff);
+    header[1+1] = (byte) ((bytesPerChecksum >>> 16) & 0xff);
+    header[1+2] = (byte) ((bytesPerChecksum >>> 8) & 0xff);
+    header[1+3] = (byte) (bytesPerChecksum & 0xff);
+    return header;
+  }
+  
+  /**
+   * Writes the current checksum to the stream.
+   * If <i>reset</i> is true, then resets the checksum.
+   * @return number of bytes written. Will be equal to getChecksumSize();
+   */
+   public int writeValue( DataOutputStream out, boolean reset )
+                          throws IOException {
+     if ( type.size <= 0 ) {
+       return 0;
+     }
+
+     if ( type.size == 4 ) {
+       out.writeInt( (int) summer.getValue() );
+     } else {
+       throw new IOException( "Unknown Checksum " + type );
+     }
+     
+     if ( reset ) {
+       reset();
+     }
+     
+     return type.size;
+   }
+   
+   /**
+    * Writes the current checksum to a buffer.
+    * If <i>reset</i> is true, then resets the checksum.
+    * @return number of bytes written. Will be equal to getChecksumSize();
+    */
+    public int writeValue( byte[] buf, int offset, boolean reset )
+                           throws IOException {
+      if ( type.size <= 0 ) {
+        return 0;
+      }
+
+      if ( type.size == 4 ) {
+        int checksum = (int) summer.getValue();
+        buf[offset+0] = (byte) ((checksum >>> 24) & 0xff);
+        buf[offset+1] = (byte) ((checksum >>> 16) & 0xff);
+        buf[offset+2] = (byte) ((checksum >>> 8) & 0xff);
+        buf[offset+3] = (byte) (checksum & 0xff);
+      } else {
+        throw new IOException( "Unknown Checksum " + type );
+      }
+      
+      if ( reset ) {
+        reset();
+      }
+      
+      return type.size;
+    }
+   
+   /**
+    * Compares the checksum located at buf[offset] with the current checksum.
+    * @return true if the checksum matches and false otherwise.
+    */
+   public boolean compare( byte buf[], int offset ) {
+     if ( type.size == 4 ) {
+       int checksum = ( (buf[offset+0] & 0xff) << 24 ) | 
+                      ( (buf[offset+1] & 0xff) << 16 ) |
+                      ( (buf[offset+2] & 0xff) << 8 )  |
+                      ( (buf[offset+3] & 0xff) );
+       return checksum == (int) summer.getValue();
+     }
+     return type.size == 0;
+   }
+   
+  private final Type type;
+  private final Checksum summer;
+  private final int bytesPerChecksum;
+  private int inSum = 0;
+  
+  private DataChecksum( Type type, Checksum checksum, int chunkSize ) {
+    this.type = type;
+    summer = checksum;
+    bytesPerChecksum = chunkSize;
+  }
+  
+  // Accessors
+  public Type getChecksumType() {
+    return type;
+  }
+  public int getChecksumSize() {
+    return type.size;
+  }
+  public int getBytesPerChecksum() {
+    return bytesPerChecksum;
+  }
+  public int getNumBytesInSum() {
+    return inSum;
+  }
+  
+  public static final int SIZE_OF_INTEGER = Integer.SIZE / Byte.SIZE;
+  static public int getChecksumHeaderSize() {
+    return 1 + SIZE_OF_INTEGER; // type byte, bytesPerChecksum int
+  }
+  //Checksum Interface. Just a wrapper around member summer.
+  @Override
+  public long getValue() {
+    return summer.getValue();
+  }
+  @Override
+  public void reset() {
+    summer.reset();
+    inSum = 0;
+  }
+  @Override
+  public void update( byte[] b, int off, int len ) {
+    if ( len > 0 ) {
+      summer.update( b, off, len );
+      inSum += len;
+    }
+  }
+  @Override
+  public void update( int b ) {
+    summer.update( b );
+    inSum += 1;
+  }
+  
+  /**
+   * Verify that the given checksums match the given data.
+   * 
+   * The 'mark' of the ByteBuffer parameters may be modified by this function,.
+   * but the position is maintained.
+   *  
+   * @param data the DirectByteBuffer pointing to the data to verify.
+   * @param checksums the DirectByteBuffer pointing to a series of stored
+   *                  checksums
+   * @param fileName the name of the file being read, for error-reporting
+   * @param basePos the file position to which the start of 'data' corresponds
+   * @throws ChecksumException if the checksums do not match
+   */
+  public void verifyChunkedSums(ByteBuffer data, ByteBuffer checksums,
+      String fileName, long basePos)
+  throws ChecksumException {
+    if (type.size == 0) return;
+    
+    if (data.hasArray() && checksums.hasArray()) {
+      verifyChunkedSums(
+          data.array(), data.arrayOffset() + data.position(), data.remaining(),
+          checksums.array(), checksums.arrayOffset() + checksums.position(),
+          fileName, basePos);
+      return;
+    }
+    if (NativeCrc32.isAvailable()) {
+      NativeCrc32.verifyChunkedSums(bytesPerChecksum, type.id, checksums, data,
+          fileName, basePos);
+      return;
+    }
+    
+    int startDataPos = data.position();
+    data.mark();
+    checksums.mark();
+    try {
+      byte[] buf = new byte[bytesPerChecksum];
+      byte[] sum = new byte[type.size];
+      while (data.remaining() > 0) {
+        int n = Math.min(data.remaining(), bytesPerChecksum);
+        checksums.get(sum);
+        data.get(buf, 0, n);
+        summer.reset();
+        summer.update(buf, 0, n);
+        int calculated = (int)summer.getValue();
+        int stored = (sum[0] << 24 & 0xff000000) |
+          (sum[1] << 16 & 0xff0000) |
+          (sum[2] << 8 & 0xff00) |
+          sum[3] & 0xff;
+        if (calculated != stored) {
+          long errPos = basePos + data.position() - startDataPos - n;
+          throw new ChecksumException(
+              "Checksum error: "+ fileName + " at "+ errPos +
+              " exp: " + stored + " got: " + calculated, errPos);
+        }
+      }
+    } finally {
+      data.reset();
+      checksums.reset();
+    }
+  }
+  
+  /**
+   * Implementation of chunked verification specifically on byte arrays. This
+   * is to avoid the copy when dealing with ByteBuffers that have array backing.
+   */
+  private void verifyChunkedSums(
+      byte[] data, int dataOff, int dataLen,
+      byte[] checksums, int checksumsOff, String fileName,
+      long basePos) throws ChecksumException {
+    
+    int remaining = dataLen;
+    int dataPos = 0;
+    while (remaining > 0) {
+      int n = Math.min(remaining, bytesPerChecksum);
+      
+      summer.reset();
+      summer.update(data, dataOff + dataPos, n);
+      dataPos += n;
+      remaining -= n;
+      
+      int calculated = (int)summer.getValue();
+      int stored = (checksums[checksumsOff] << 24 & 0xff000000) |
+        (checksums[checksumsOff + 1] << 16 & 0xff0000) |
+        (checksums[checksumsOff + 2] << 8 & 0xff00) |
+        checksums[checksumsOff + 3] & 0xff;
+      checksumsOff += 4;
+      if (calculated != stored) {
+        long errPos = basePos + dataPos - n;
+        throw new ChecksumException(
+            "Checksum error: "+ fileName + " at "+ errPos +
+            " exp: " + stored + " got: " + calculated, errPos);
+      }
+    }
+  }
+
+  /**
+   * Calculate checksums for the given data.
+   * 
+   * The 'mark' of the ByteBuffer parameters may be modified by this function,
+   * but the position is maintained.
+   * 
+   * @param data the DirectByteBuffer pointing to the data to checksum.
+   * @param checksums the DirectByteBuffer into which checksums will be
+   *                  stored. Enough space must be available in this
+   *                  buffer to put the checksums.
+   */
+  public void calculateChunkedSums(ByteBuffer data, ByteBuffer checksums) {
+    if (type.size == 0) return;
+    
+    if (data.hasArray() && checksums.hasArray()) {
+      calculateChunkedSums(data.array(), data.arrayOffset() + data.position(), data.remaining(),
+          checksums.array(), checksums.arrayOffset() + checksums.position());
+      return;
+    }
+    
+    data.mark();
+    checksums.mark();
+    try {
+      byte[] buf = new byte[bytesPerChecksum];
+      while (data.remaining() > 0) {
+        int n = Math.min(data.remaining(), bytesPerChecksum);
+        data.get(buf, 0, n);
+        summer.reset();
+        summer.update(buf, 0, n);
+        checksums.putInt((int)summer.getValue());
+      }
+    } finally {
+      data.reset();
+      checksums.reset();
+    }
+  }
+
+  /**
+   * Implementation of chunked calculation specifically on byte arrays. This
+   * is to avoid the copy when dealing with ByteBuffers that have array backing.
+   */
+  private void calculateChunkedSums(
+      byte[] data, int dataOffset, int dataLength,
+      byte[] sums, int sumsOffset) {
+
+    int remaining = dataLength;
+    while (remaining > 0) {
+      int n = Math.min(remaining, bytesPerChecksum);
+      summer.reset();
+      summer.update(data, dataOffset, n);
+      dataOffset += n;
+      remaining -= n;
+      long calculated = summer.getValue();
+      sums[sumsOffset++] = (byte) (calculated >> 24);
+      sums[sumsOffset++] = (byte) (calculated >> 16);
+      sums[sumsOffset++] = (byte) (calculated >> 8);
+      sums[sumsOffset++] = (byte) (calculated);
+    }
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (!(other instanceof DataChecksum)) {
+      return false;
+    }
+    DataChecksum o = (DataChecksum)other;
+    return o.bytesPerChecksum == this.bytesPerChecksum &&
+      o.type == this.type;
+  }
+  
+  @Override
+  public int hashCode() {
+    return (this.type.id + 31) * this.bytesPerChecksum;
+  }
+  
+  @Override
+  public String toString() {
+    return "DataChecksum(type=" + type +
+      ", chunkSize=" + bytesPerChecksum + ")";
+  }
+  
+  /**
+   * This just provides a dummy implimentation for Checksum class
+   * This is used when there is no checksum available or required for 
+   * data
+   */
+  static class ChecksumNull implements Checksum {
+    
+    public ChecksumNull() {}
+    
+    //Dummy interface
+    @Override
+    public long getValue() { return 0; }
+    @Override
+    public void reset() {}
+    @Override
+    public void update(byte[] b, int off, int len) {}
+    @Override
+    public void update(int b) {}
+  };
+}



Mime
View raw message