hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1401071 [4/4] - in /hadoop/common/branches/branch-trunk-win/hadoop-common-project: hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/ hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/ hado...
Date Mon, 22 Oct 2012 20:43:25 GMT
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java Mon Oct 22 20:43:16 2012
@@ -1,460 +1,460 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.util;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.zip.Checksum;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.ChecksumException;
-
-/**
- * This class provides inteface and utilities for processing checksums for
- * DFS data transfers.
- */
-@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
-@InterfaceStability.Evolving
-public class DataChecksum implements Checksum {
-  
-  // Misc constants
-  public static final int HEADER_LEN = 5; /// 1 byte type and 4 byte len
-  
-  // checksum types
-  public static final int CHECKSUM_NULL    = 0;
-  public static final int CHECKSUM_CRC32   = 1;
-  public static final int CHECKSUM_CRC32C  = 2;
-  public static final int CHECKSUM_DEFAULT = 3; 
-  public static final int CHECKSUM_MIXED   = 4;
- 
-  /** The checksum types */
-  public static enum Type {
-    NULL  (CHECKSUM_NULL, 0),
-    CRC32 (CHECKSUM_CRC32, 4),
-    CRC32C(CHECKSUM_CRC32C, 4),
-    DEFAULT(CHECKSUM_DEFAULT, 0), // This cannot be used to create DataChecksum
-    MIXED (CHECKSUM_MIXED, 0); // This cannot be used to create DataChecksum
-
-    public final int id;
-    public final int size;
-    
-    private Type(int id, int size) {
-      this.id = id;
-      this.size = size;
-    }
-
-    /** @return the type corresponding to the id. */
-    public static Type valueOf(int id) {
-      if (id < 0 || id >= values().length) {
-        throw new IllegalArgumentException("id=" + id
-            + " out of range [0, " + values().length + ")");
-      }
-      return values()[id];
-    }
-  }
-
-
-  public static DataChecksum newDataChecksum(Type type, int bytesPerChecksum ) {
-    if ( bytesPerChecksum <= 0 ) {
-      return null;
-    }
-    
-    switch ( type ) {
-    case NULL :
-      return new DataChecksum(type, new ChecksumNull(), bytesPerChecksum );
-    case CRC32 :
-      return new DataChecksum(type, new PureJavaCrc32(), bytesPerChecksum );
-    case CRC32C:
-      return new DataChecksum(type, new PureJavaCrc32C(), bytesPerChecksum);
-    default:
-      return null;  
-    }
-  }
-  
-  /**
-   * Creates a DataChecksum from HEADER_LEN bytes from arr[offset].
-   * @return DataChecksum of the type in the array or null in case of an error.
-   */
-  public static DataChecksum newDataChecksum( byte bytes[], int offset ) {
-    if ( offset < 0 || bytes.length < offset + HEADER_LEN ) {
-      return null;
-    }
-    
-    // like readInt():
-    int bytesPerChecksum = ( (bytes[offset+1] & 0xff) << 24 ) | 
-                           ( (bytes[offset+2] & 0xff) << 16 ) |
-                           ( (bytes[offset+3] & 0xff) << 8 )  |
-                           ( (bytes[offset+4] & 0xff) );
-    return newDataChecksum( Type.valueOf(bytes[0]), bytesPerChecksum );
-  }
-  
-  /**
-   * This constructucts a DataChecksum by reading HEADER_LEN bytes from
-   * input stream <i>in</i>
-   */
-  public static DataChecksum newDataChecksum( DataInputStream in )
-                                 throws IOException {
-    int type = in.readByte();
-    int bpc = in.readInt();
-    DataChecksum summer = newDataChecksum(Type.valueOf(type), bpc );
-    if ( summer == null ) {
-      throw new IOException( "Could not create DataChecksum of type " +
-                             type + " with bytesPerChecksum " + bpc );
-    }
-    return summer;
-  }
-  
-  /**
-   * Writes the checksum header to the output stream <i>out</i>.
-   */
-  public void writeHeader( DataOutputStream out ) 
-                           throws IOException { 
-    out.writeByte( type.id );
-    out.writeInt( bytesPerChecksum );
-  }
-
-  public byte[] getHeader() {
-    byte[] header = new byte[DataChecksum.HEADER_LEN];
-    header[0] = (byte) (type.id & 0xff);
-    // Writing in buffer just like DataOutput.WriteInt()
-    header[1+0] = (byte) ((bytesPerChecksum >>> 24) & 0xff);
-    header[1+1] = (byte) ((bytesPerChecksum >>> 16) & 0xff);
-    header[1+2] = (byte) ((bytesPerChecksum >>> 8) & 0xff);
-    header[1+3] = (byte) (bytesPerChecksum & 0xff);
-    return header;
-  }
-  
-  /**
-   * Writes the current checksum to the stream.
-   * If <i>reset</i> is true, then resets the checksum.
-   * @return number of bytes written. Will be equal to getChecksumSize();
-   */
-   public int writeValue( DataOutputStream out, boolean reset )
-                          throws IOException {
-     if ( type.size <= 0 ) {
-       return 0;
-     }
-
-     if ( type.size == 4 ) {
-       out.writeInt( (int) summer.getValue() );
-     } else {
-       throw new IOException( "Unknown Checksum " + type );
-     }
-     
-     if ( reset ) {
-       reset();
-     }
-     
-     return type.size;
-   }
-   
-   /**
-    * Writes the current checksum to a buffer.
-    * If <i>reset</i> is true, then resets the checksum.
-    * @return number of bytes written. Will be equal to getChecksumSize();
-    */
-    public int writeValue( byte[] buf, int offset, boolean reset )
-                           throws IOException {
-      if ( type.size <= 0 ) {
-        return 0;
-      }
-
-      if ( type.size == 4 ) {
-        int checksum = (int) summer.getValue();
-        buf[offset+0] = (byte) ((checksum >>> 24) & 0xff);
-        buf[offset+1] = (byte) ((checksum >>> 16) & 0xff);
-        buf[offset+2] = (byte) ((checksum >>> 8) & 0xff);
-        buf[offset+3] = (byte) (checksum & 0xff);
-      } else {
-        throw new IOException( "Unknown Checksum " + type );
-      }
-      
-      if ( reset ) {
-        reset();
-      }
-      
-      return type.size;
-    }
-   
-   /**
-    * Compares the checksum located at buf[offset] with the current checksum.
-    * @return true if the checksum matches and false otherwise.
-    */
-   public boolean compare( byte buf[], int offset ) {
-     if ( type.size == 4 ) {
-       int checksum = ( (buf[offset+0] & 0xff) << 24 ) | 
-                      ( (buf[offset+1] & 0xff) << 16 ) |
-                      ( (buf[offset+2] & 0xff) << 8 )  |
-                      ( (buf[offset+3] & 0xff) );
-       return checksum == (int) summer.getValue();
-     }
-     return type.size == 0;
-   }
-   
-  private final Type type;
-  private final Checksum summer;
-  private final int bytesPerChecksum;
-  private int inSum = 0;
-  
-  private DataChecksum( Type type, Checksum checksum, int chunkSize ) {
-    this.type = type;
-    summer = checksum;
-    bytesPerChecksum = chunkSize;
-  }
-  
-  // Accessors
-  public Type getChecksumType() {
-    return type;
-  }
-  public int getChecksumSize() {
-    return type.size;
-  }
-  public int getBytesPerChecksum() {
-    return bytesPerChecksum;
-  }
-  public int getNumBytesInSum() {
-    return inSum;
-  }
-  
-  public static final int SIZE_OF_INTEGER = Integer.SIZE / Byte.SIZE;
-  static public int getChecksumHeaderSize() {
-    return 1 + SIZE_OF_INTEGER; // type byte, bytesPerChecksum int
-  }
-  //Checksum Interface. Just a wrapper around member summer.
-  @Override
-  public long getValue() {
-    return summer.getValue();
-  }
-  @Override
-  public void reset() {
-    summer.reset();
-    inSum = 0;
-  }
-  @Override
-  public void update( byte[] b, int off, int len ) {
-    if ( len > 0 ) {
-      summer.update( b, off, len );
-      inSum += len;
-    }
-  }
-  @Override
-  public void update( int b ) {
-    summer.update( b );
-    inSum += 1;
-  }
-  
-  /**
-   * Verify that the given checksums match the given data.
-   * 
-   * The 'mark' of the ByteBuffer parameters may be modified by this function,.
-   * but the position is maintained.
-   *  
-   * @param data the DirectByteBuffer pointing to the data to verify.
-   * @param checksums the DirectByteBuffer pointing to a series of stored
-   *                  checksums
-   * @param fileName the name of the file being read, for error-reporting
-   * @param basePos the file position to which the start of 'data' corresponds
-   * @throws ChecksumException if the checksums do not match
-   */
-  public void verifyChunkedSums(ByteBuffer data, ByteBuffer checksums,
-      String fileName, long basePos)
-  throws ChecksumException {
-    if (type.size == 0) return;
-    
-    if (data.hasArray() && checksums.hasArray()) {
-      verifyChunkedSums(
-          data.array(), data.arrayOffset() + data.position(), data.remaining(),
-          checksums.array(), checksums.arrayOffset() + checksums.position(),
-          fileName, basePos);
-      return;
-    }
-    if (NativeCrc32.isAvailable()) {
-      NativeCrc32.verifyChunkedSums(bytesPerChecksum, type.id, checksums, data,
-          fileName, basePos);
-      return;
-    }
-    
-    int startDataPos = data.position();
-    data.mark();
-    checksums.mark();
-    try {
-      byte[] buf = new byte[bytesPerChecksum];
-      byte[] sum = new byte[type.size];
-      while (data.remaining() > 0) {
-        int n = Math.min(data.remaining(), bytesPerChecksum);
-        checksums.get(sum);
-        data.get(buf, 0, n);
-        summer.reset();
-        summer.update(buf, 0, n);
-        int calculated = (int)summer.getValue();
-        int stored = (sum[0] << 24 & 0xff000000) |
-          (sum[1] << 16 & 0xff0000) |
-          (sum[2] << 8 & 0xff00) |
-          sum[3] & 0xff;
-        if (calculated != stored) {
-          long errPos = basePos + data.position() - startDataPos - n;
-          throw new ChecksumException(
-              "Checksum error: "+ fileName + " at "+ errPos +
-              " exp: " + stored + " got: " + calculated, errPos);
-        }
-      }
-    } finally {
-      data.reset();
-      checksums.reset();
-    }
-  }
-  
-  /**
-   * Implementation of chunked verification specifically on byte arrays. This
-   * is to avoid the copy when dealing with ByteBuffers that have array backing.
-   */
-  private void verifyChunkedSums(
-      byte[] data, int dataOff, int dataLen,
-      byte[] checksums, int checksumsOff, String fileName,
-      long basePos) throws ChecksumException {
-    
-    int remaining = dataLen;
-    int dataPos = 0;
-    while (remaining > 0) {
-      int n = Math.min(remaining, bytesPerChecksum);
-      
-      summer.reset();
-      summer.update(data, dataOff + dataPos, n);
-      dataPos += n;
-      remaining -= n;
-      
-      int calculated = (int)summer.getValue();
-      int stored = (checksums[checksumsOff] << 24 & 0xff000000) |
-        (checksums[checksumsOff + 1] << 16 & 0xff0000) |
-        (checksums[checksumsOff + 2] << 8 & 0xff00) |
-        checksums[checksumsOff + 3] & 0xff;
-      checksumsOff += 4;
-      if (calculated != stored) {
-        long errPos = basePos + dataPos - n;
-        throw new ChecksumException(
-            "Checksum error: "+ fileName + " at "+ errPos +
-            " exp: " + stored + " got: " + calculated, errPos);
-      }
-    }
-  }
-
-  /**
-   * Calculate checksums for the given data.
-   * 
-   * The 'mark' of the ByteBuffer parameters may be modified by this function,
-   * but the position is maintained.
-   * 
-   * @param data the DirectByteBuffer pointing to the data to checksum.
-   * @param checksums the DirectByteBuffer into which checksums will be
-   *                  stored. Enough space must be available in this
-   *                  buffer to put the checksums.
-   */
-  public void calculateChunkedSums(ByteBuffer data, ByteBuffer checksums) {
-    if (type.size == 0) return;
-    
-    if (data.hasArray() && checksums.hasArray()) {
-      calculateChunkedSums(data.array(), data.arrayOffset() + data.position(), data.remaining(),
-          checksums.array(), checksums.arrayOffset() + checksums.position());
-      return;
-    }
-    
-    data.mark();
-    checksums.mark();
-    try {
-      byte[] buf = new byte[bytesPerChecksum];
-      while (data.remaining() > 0) {
-        int n = Math.min(data.remaining(), bytesPerChecksum);
-        data.get(buf, 0, n);
-        summer.reset();
-        summer.update(buf, 0, n);
-        checksums.putInt((int)summer.getValue());
-      }
-    } finally {
-      data.reset();
-      checksums.reset();
-    }
-  }
-
-  /**
-   * Implementation of chunked calculation specifically on byte arrays. This
-   * is to avoid the copy when dealing with ByteBuffers that have array backing.
-   */
-  private void calculateChunkedSums(
-      byte[] data, int dataOffset, int dataLength,
-      byte[] sums, int sumsOffset) {
-
-    int remaining = dataLength;
-    while (remaining > 0) {
-      int n = Math.min(remaining, bytesPerChecksum);
-      summer.reset();
-      summer.update(data, dataOffset, n);
-      dataOffset += n;
-      remaining -= n;
-      long calculated = summer.getValue();
-      sums[sumsOffset++] = (byte) (calculated >> 24);
-      sums[sumsOffset++] = (byte) (calculated >> 16);
-      sums[sumsOffset++] = (byte) (calculated >> 8);
-      sums[sumsOffset++] = (byte) (calculated);
-    }
-  }
-
-  @Override
-  public boolean equals(Object other) {
-    if (!(other instanceof DataChecksum)) {
-      return false;
-    }
-    DataChecksum o = (DataChecksum)other;
-    return o.bytesPerChecksum == this.bytesPerChecksum &&
-      o.type == this.type;
-  }
-  
-  @Override
-  public int hashCode() {
-    return (this.type.id + 31) * this.bytesPerChecksum;
-  }
-  
-  @Override
-  public String toString() {
-    return "DataChecksum(type=" + type +
-      ", chunkSize=" + bytesPerChecksum + ")";
-  }
-  
-  /**
-   * This just provides a dummy implimentation for Checksum class
-   * This is used when there is no checksum available or required for 
-   * data
-   */
-  static class ChecksumNull implements Checksum {
-    
-    public ChecksumNull() {}
-    
-    //Dummy interface
-    @Override
-    public long getValue() { return 0; }
-    @Override
-    public void reset() {}
-    @Override
-    public void update(byte[] b, int off, int len) {}
-    @Override
-    public void update(int b) {}
-  };
-}
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.zip.Checksum;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.ChecksumException;
+
+/**
+ * This class provides inteface and utilities for processing checksums for
+ * DFS data transfers.
+ */
+@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
+@InterfaceStability.Evolving
+public class DataChecksum implements Checksum {
+  
+  // Misc constants
+  public static final int HEADER_LEN = 5; /// 1 byte type and 4 byte len
+  
+  // checksum types
+  public static final int CHECKSUM_NULL    = 0;
+  public static final int CHECKSUM_CRC32   = 1;
+  public static final int CHECKSUM_CRC32C  = 2;
+  public static final int CHECKSUM_DEFAULT = 3; 
+  public static final int CHECKSUM_MIXED   = 4;
+ 
+  /** The checksum types */
+  public static enum Type {
+    NULL  (CHECKSUM_NULL, 0),
+    CRC32 (CHECKSUM_CRC32, 4),
+    CRC32C(CHECKSUM_CRC32C, 4),
+    DEFAULT(CHECKSUM_DEFAULT, 0), // This cannot be used to create DataChecksum
+    MIXED (CHECKSUM_MIXED, 0); // This cannot be used to create DataChecksum
+
+    public final int id;
+    public final int size;
+    
+    private Type(int id, int size) {
+      this.id = id;
+      this.size = size;
+    }
+
+    /** @return the type corresponding to the id. */
+    public static Type valueOf(int id) {
+      if (id < 0 || id >= values().length) {
+        throw new IllegalArgumentException("id=" + id
+            + " out of range [0, " + values().length + ")");
+      }
+      return values()[id];
+    }
+  }
+
+
+  public static DataChecksum newDataChecksum(Type type, int bytesPerChecksum ) {
+    if ( bytesPerChecksum <= 0 ) {
+      return null;
+    }
+    
+    switch ( type ) {
+    case NULL :
+      return new DataChecksum(type, new ChecksumNull(), bytesPerChecksum );
+    case CRC32 :
+      return new DataChecksum(type, new PureJavaCrc32(), bytesPerChecksum );
+    case CRC32C:
+      return new DataChecksum(type, new PureJavaCrc32C(), bytesPerChecksum);
+    default:
+      return null;  
+    }
+  }
+  
+  /**
+   * Creates a DataChecksum from HEADER_LEN bytes from arr[offset].
+   * @return DataChecksum of the type in the array or null in case of an error.
+   */
+  public static DataChecksum newDataChecksum( byte bytes[], int offset ) {
+    if ( offset < 0 || bytes.length < offset + HEADER_LEN ) {
+      return null;
+    }
+    
+    // like readInt():
+    int bytesPerChecksum = ( (bytes[offset+1] & 0xff) << 24 ) | 
+                           ( (bytes[offset+2] & 0xff) << 16 ) |
+                           ( (bytes[offset+3] & 0xff) << 8 )  |
+                           ( (bytes[offset+4] & 0xff) );
+    return newDataChecksum( Type.valueOf(bytes[0]), bytesPerChecksum );
+  }
+  
+  /**
+   * This constructucts a DataChecksum by reading HEADER_LEN bytes from
+   * input stream <i>in</i>
+   */
+  public static DataChecksum newDataChecksum( DataInputStream in )
+                                 throws IOException {
+    int type = in.readByte();
+    int bpc = in.readInt();
+    DataChecksum summer = newDataChecksum(Type.valueOf(type), bpc );
+    if ( summer == null ) {
+      throw new IOException( "Could not create DataChecksum of type " +
+                             type + " with bytesPerChecksum " + bpc );
+    }
+    return summer;
+  }
+  
+  /**
+   * Writes the checksum header to the output stream <i>out</i>.
+   */
+  public void writeHeader( DataOutputStream out ) 
+                           throws IOException { 
+    out.writeByte( type.id );
+    out.writeInt( bytesPerChecksum );
+  }
+
+  public byte[] getHeader() {
+    byte[] header = new byte[DataChecksum.HEADER_LEN];
+    header[0] = (byte) (type.id & 0xff);
+    // Writing in buffer just like DataOutput.WriteInt()
+    header[1+0] = (byte) ((bytesPerChecksum >>> 24) & 0xff);
+    header[1+1] = (byte) ((bytesPerChecksum >>> 16) & 0xff);
+    header[1+2] = (byte) ((bytesPerChecksum >>> 8) & 0xff);
+    header[1+3] = (byte) (bytesPerChecksum & 0xff);
+    return header;
+  }
+  
+  /**
+   * Writes the current checksum to the stream.
+   * If <i>reset</i> is true, then resets the checksum.
+   * @return number of bytes written. Will be equal to getChecksumSize();
+   */
+   public int writeValue( DataOutputStream out, boolean reset )
+                          throws IOException {
+     if ( type.size <= 0 ) {
+       return 0;
+     }
+
+     if ( type.size == 4 ) {
+       out.writeInt( (int) summer.getValue() );
+     } else {
+       throw new IOException( "Unknown Checksum " + type );
+     }
+     
+     if ( reset ) {
+       reset();
+     }
+     
+     return type.size;
+   }
+   
+   /**
+    * Writes the current checksum to a buffer.
+    * If <i>reset</i> is true, then resets the checksum.
+    * @return number of bytes written. Will be equal to getChecksumSize();
+    */
+    public int writeValue( byte[] buf, int offset, boolean reset )
+                           throws IOException {
+      if ( type.size <= 0 ) {
+        return 0;
+      }
+
+      if ( type.size == 4 ) {
+        int checksum = (int) summer.getValue();
+        buf[offset+0] = (byte) ((checksum >>> 24) & 0xff);
+        buf[offset+1] = (byte) ((checksum >>> 16) & 0xff);
+        buf[offset+2] = (byte) ((checksum >>> 8) & 0xff);
+        buf[offset+3] = (byte) (checksum & 0xff);
+      } else {
+        throw new IOException( "Unknown Checksum " + type );
+      }
+      
+      if ( reset ) {
+        reset();
+      }
+      
+      return type.size;
+    }
+   
+   /**
+    * Compares the checksum located at buf[offset] with the current checksum.
+    * @return true if the checksum matches and false otherwise.
+    */
+   public boolean compare( byte buf[], int offset ) {
+     if ( type.size == 4 ) {
+       int checksum = ( (buf[offset+0] & 0xff) << 24 ) | 
+                      ( (buf[offset+1] & 0xff) << 16 ) |
+                      ( (buf[offset+2] & 0xff) << 8 )  |
+                      ( (buf[offset+3] & 0xff) );
+       return checksum == (int) summer.getValue();
+     }
+     return type.size == 0;
+   }
+   
+  private final Type type;
+  private final Checksum summer;
+  private final int bytesPerChecksum;
+  private int inSum = 0;
+  
+  private DataChecksum( Type type, Checksum checksum, int chunkSize ) {
+    this.type = type;
+    summer = checksum;
+    bytesPerChecksum = chunkSize;
+  }
+  
+  // Accessors
+  public Type getChecksumType() {
+    return type;
+  }
+  public int getChecksumSize() {
+    return type.size;
+  }
+  public int getBytesPerChecksum() {
+    return bytesPerChecksum;
+  }
+  public int getNumBytesInSum() {
+    return inSum;
+  }
+  
+  public static final int SIZE_OF_INTEGER = Integer.SIZE / Byte.SIZE;
+  static public int getChecksumHeaderSize() {
+    return 1 + SIZE_OF_INTEGER; // type byte, bytesPerChecksum int
+  }
+  //Checksum Interface. Just a wrapper around member summer.
+  @Override
+  public long getValue() {
+    return summer.getValue();
+  }
+  @Override
+  public void reset() {
+    summer.reset();
+    inSum = 0;
+  }
+  @Override
+  public void update( byte[] b, int off, int len ) {
+    if ( len > 0 ) {
+      summer.update( b, off, len );
+      inSum += len;
+    }
+  }
+  @Override
+  public void update( int b ) {
+    summer.update( b );
+    inSum += 1;
+  }
+  
+  /**
+   * Verify that the given checksums match the given data.
+   * 
+   * The 'mark' of the ByteBuffer parameters may be modified by this function,.
+   * but the position is maintained.
+   *  
+   * @param data the DirectByteBuffer pointing to the data to verify.
+   * @param checksums the DirectByteBuffer pointing to a series of stored
+   *                  checksums
+   * @param fileName the name of the file being read, for error-reporting
+   * @param basePos the file position to which the start of 'data' corresponds
+   * @throws ChecksumException if the checksums do not match
+   */
+  public void verifyChunkedSums(ByteBuffer data, ByteBuffer checksums,
+      String fileName, long basePos)
+  throws ChecksumException {
+    if (type.size == 0) return;
+    
+    if (data.hasArray() && checksums.hasArray()) {
+      verifyChunkedSums(
+          data.array(), data.arrayOffset() + data.position(), data.remaining(),
+          checksums.array(), checksums.arrayOffset() + checksums.position(),
+          fileName, basePos);
+      return;
+    }
+    if (NativeCrc32.isAvailable()) {
+      NativeCrc32.verifyChunkedSums(bytesPerChecksum, type.id, checksums, data,
+          fileName, basePos);
+      return;
+    }
+    
+    int startDataPos = data.position();
+    data.mark();
+    checksums.mark();
+    try {
+      byte[] buf = new byte[bytesPerChecksum];
+      byte[] sum = new byte[type.size];
+      while (data.remaining() > 0) {
+        int n = Math.min(data.remaining(), bytesPerChecksum);
+        checksums.get(sum);
+        data.get(buf, 0, n);
+        summer.reset();
+        summer.update(buf, 0, n);
+        int calculated = (int)summer.getValue();
+        int stored = (sum[0] << 24 & 0xff000000) |
+          (sum[1] << 16 & 0xff0000) |
+          (sum[2] << 8 & 0xff00) |
+          sum[3] & 0xff;
+        if (calculated != stored) {
+          long errPos = basePos + data.position() - startDataPos - n;
+          throw new ChecksumException(
+              "Checksum error: "+ fileName + " at "+ errPos +
+              " exp: " + stored + " got: " + calculated, errPos);
+        }
+      }
+    } finally {
+      data.reset();
+      checksums.reset();
+    }
+  }
+  
+  /**
+   * Implementation of chunked verification specifically on byte arrays. This
+   * is to avoid the copy when dealing with ByteBuffers that have array backing.
+   */
+  private void verifyChunkedSums(
+      byte[] data, int dataOff, int dataLen,
+      byte[] checksums, int checksumsOff, String fileName,
+      long basePos) throws ChecksumException {
+    
+    int remaining = dataLen;
+    int dataPos = 0;
+    while (remaining > 0) {
+      int n = Math.min(remaining, bytesPerChecksum);
+      
+      summer.reset();
+      summer.update(data, dataOff + dataPos, n);
+      dataPos += n;
+      remaining -= n;
+      
+      int calculated = (int)summer.getValue();
+      int stored = (checksums[checksumsOff] << 24 & 0xff000000) |
+        (checksums[checksumsOff + 1] << 16 & 0xff0000) |
+        (checksums[checksumsOff + 2] << 8 & 0xff00) |
+        checksums[checksumsOff + 3] & 0xff;
+      checksumsOff += 4;
+      if (calculated != stored) {
+        long errPos = basePos + dataPos - n;
+        throw new ChecksumException(
+            "Checksum error: "+ fileName + " at "+ errPos +
+            " exp: " + stored + " got: " + calculated, errPos);
+      }
+    }
+  }
+
+  /**
+   * Calculate checksums for the given data.
+   * 
+   * The 'mark' of the ByteBuffer parameters may be modified by this function,
+   * but the position is maintained.
+   * 
+   * @param data the DirectByteBuffer pointing to the data to checksum.
+   * @param checksums the DirectByteBuffer into which checksums will be
+   *                  stored. Enough space must be available in this
+   *                  buffer to put the checksums.
+   */
+  public void calculateChunkedSums(ByteBuffer data, ByteBuffer checksums) {
+    if (type.size == 0) return;
+    
+    if (data.hasArray() && checksums.hasArray()) {
+      calculateChunkedSums(data.array(), data.arrayOffset() + data.position(), data.remaining(),
+          checksums.array(), checksums.arrayOffset() + checksums.position());
+      return;
+    }
+    
+    data.mark();
+    checksums.mark();
+    try {
+      byte[] buf = new byte[bytesPerChecksum];
+      while (data.remaining() > 0) {
+        int n = Math.min(data.remaining(), bytesPerChecksum);
+        data.get(buf, 0, n);
+        summer.reset();
+        summer.update(buf, 0, n);
+        checksums.putInt((int)summer.getValue());
+      }
+    } finally {
+      data.reset();
+      checksums.reset();
+    }
+  }
+
+  /**
+   * Implementation of chunked calculation specifically on byte arrays. This
+   * is to avoid the copy when dealing with ByteBuffers that have array backing.
+   */
+  private void calculateChunkedSums(
+      byte[] data, int dataOffset, int dataLength,
+      byte[] sums, int sumsOffset) {
+
+    int remaining = dataLength;
+    while (remaining > 0) {
+      int n = Math.min(remaining, bytesPerChecksum);
+      summer.reset();
+      summer.update(data, dataOffset, n);
+      dataOffset += n;
+      remaining -= n;
+      long calculated = summer.getValue();
+      sums[sumsOffset++] = (byte) (calculated >> 24);
+      sums[sumsOffset++] = (byte) (calculated >> 16);
+      sums[sumsOffset++] = (byte) (calculated >> 8);
+      sums[sumsOffset++] = (byte) (calculated);
+    }
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (!(other instanceof DataChecksum)) {
+      return false;
+    }
+    DataChecksum o = (DataChecksum)other;
+    return o.bytesPerChecksum == this.bytesPerChecksum &&
+      o.type == this.type;
+  }
+  
+  @Override
+  public int hashCode() {
+    return (this.type.id + 31) * this.bytesPerChecksum;
+  }
+  
+  @Override
+  public String toString() {
+    return "DataChecksum(type=" + type +
+      ", chunkSize=" + bytesPerChecksum + ")";
+  }
+  
+  /**
+   * This just provides a dummy implimentation for Checksum class
+   * This is used when there is no checksum available or required for 
+   * data
+   */
+  static class ChecksumNull implements Checksum {
+    
+    public ChecksumNull() {}
+    
+    //Dummy interface
+    @Override
+    public long getValue() { return 0; }
+    @Override
+    public void reset() {}
+    @Override
+    public void update(byte[] b, int off, int len) {}
+    @Override
+    public void update(int b) {}
+  };
+}

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java Mon Oct 22 20:43:16 2012
@@ -59,22 +59,38 @@ public class PureJavaCrc32 implements Ch
   @Override
   public void update(byte[] b, int off, int len) {
     int localCrc = crc;
+
     while(len > 7) {
-      int c0 = b[off++] ^ localCrc;
-      int c1 = b[off++] ^ (localCrc >>>= 8);
-      int c2 = b[off++] ^ (localCrc >>>= 8);
-      int c3 = b[off++] ^ (localCrc >>>= 8);
-      localCrc = (T8_7[c0 & 0xff] ^ T8_6[c1 & 0xff])
-          ^ (T8_5[c2 & 0xff] ^ T8_4[c3 & 0xff]);
+      final int c0 =(b[off+0] ^ localCrc) & 0xff;
+      final int c1 =(b[off+1] ^ (localCrc >>>= 8)) & 0xff;
+      final int c2 =(b[off+2] ^ (localCrc >>>= 8)) & 0xff;
+      final int c3 =(b[off+3] ^ (localCrc >>>= 8)) & 0xff;
+      localCrc = (T[T8_7_start + c0] ^ T[T8_6_start + c1])
+          ^ (T[T8_5_start + c2] ^ T[T8_4_start + c3]);
+
+      final int c4 = b[off+4] & 0xff;
+      final int c5 = b[off+5] & 0xff;
+      final int c6 = b[off+6] & 0xff;
+      final int c7 = b[off+7] & 0xff;
 
-      localCrc ^= (T8_3[b[off++] & 0xff] ^ T8_2[b[off++] & 0xff])
-           ^ (T8_1[b[off++] & 0xff] ^ T8_0[b[off++] & 0xff]);
+      localCrc ^= (T[T8_3_start + c4] ^ T[T8_2_start + c5])
+           ^ (T[T8_1_start + c6] ^ T[T8_0_start + c7]);
 
+      off += 8;
       len -= 8;
     }
-    while(len > 0) {
-      localCrc = (localCrc >>> 8) ^ T8_0[(localCrc ^ b[off++]) & 0xff];
-      len--;
+
+    /* loop unroll - duff's device style */
+    switch(len) {
+      case 7: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 6: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 5: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 4: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 3: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 2: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 1: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      default:
+        /* nothing */
     }
     
     // Publish crc out to object
@@ -83,14 +99,24 @@ public class PureJavaCrc32 implements Ch
 
   @Override
   final public void update(int b) {
-    crc = (crc >>> 8) ^ T8_0[(crc ^ b) & 0xff];
+    crc = (crc >>> 8) ^ T[T8_0_start + ((crc ^ b) & 0xff)];
   }
 
   /*
    * CRC-32 lookup tables generated by the polynomial 0xEDB88320.
    * See also TestPureJavaCrc32.Table.
    */
-  private static final int[] T8_0 = new int[] {
+  private static final int T8_0_start = 0*256;
+  private static final int T8_1_start = 1*256;
+  private static final int T8_2_start = 2*256;
+  private static final int T8_3_start = 3*256;
+  private static final int T8_4_start = 4*256;
+  private static final int T8_5_start = 5*256;
+  private static final int T8_6_start = 6*256;
+  private static final int T8_7_start = 7*256;
+
+  private static final int[] T = new int[] {
+  	/* T8_0 */
     0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 
     0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3, 
     0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, 
@@ -154,9 +180,8 @@ public class PureJavaCrc32 implements Ch
     0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 
     0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF, 
     0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, 
-    0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D
-  };
-  private static final int[] T8_1 = new int[] {
+    0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D,
+    /* T8_1 */
     0x00000000, 0x191B3141, 0x32366282, 0x2B2D53C3, 
     0x646CC504, 0x7D77F445, 0x565AA786, 0x4F4196C7, 
     0xC8D98A08, 0xD1C2BB49, 0xFAEFE88A, 0xE3F4D9CB, 
@@ -220,9 +245,8 @@ public class PureJavaCrc32 implements Ch
     0x14BCE1BD, 0x0DA7D0FC, 0x268A833F, 0x3F91B27E, 
     0x70D024B9, 0x69CB15F8, 0x42E6463B, 0x5BFD777A, 
     0xDC656BB5, 0xC57E5AF4, 0xEE530937, 0xF7483876, 
-    0xB809AEB1, 0xA1129FF0, 0x8A3FCC33, 0x9324FD72
-  };
-  private static final int[] T8_2 = new int[] {
+    0xB809AEB1, 0xA1129FF0, 0x8A3FCC33, 0x9324FD72,
+    /* T8_2 */
     0x00000000, 0x01C26A37, 0x0384D46E, 0x0246BE59, 
     0x0709A8DC, 0x06CBC2EB, 0x048D7CB2, 0x054F1685, 
     0x0E1351B8, 0x0FD13B8F, 0x0D9785D6, 0x0C55EFE1, 
@@ -286,9 +310,8 @@ public class PureJavaCrc32 implements Ch
     0xB5C473D0, 0xB40619E7, 0xB640A7BE, 0xB782CD89, 
     0xB2CDDB0C, 0xB30FB13B, 0xB1490F62, 0xB08B6555, 
     0xBBD72268, 0xBA15485F, 0xB853F606, 0xB9919C31, 
-    0xBCDE8AB4, 0xBD1CE083, 0xBF5A5EDA, 0xBE9834ED
-  };
-  private static final int[] T8_3 = new int[] {
+    0xBCDE8AB4, 0xBD1CE083, 0xBF5A5EDA, 0xBE9834ED,
+    /* T8_3 */
     0x00000000, 0xB8BC6765, 0xAA09C88B, 0x12B5AFEE, 
     0x8F629757, 0x37DEF032, 0x256B5FDC, 0x9DD738B9, 
     0xC5B428EF, 0x7D084F8A, 0x6FBDE064, 0xD7018701, 
@@ -352,9 +375,8 @@ public class PureJavaCrc32 implements Ch
     0x866616A7, 0x3EDA71C2, 0x2C6FDE2C, 0x94D3B949, 
     0x090481F0, 0xB1B8E695, 0xA30D497B, 0x1BB12E1E, 
     0x43D23E48, 0xFB6E592D, 0xE9DBF6C3, 0x516791A6, 
-    0xCCB0A91F, 0x740CCE7A, 0x66B96194, 0xDE0506F1
-  };
-  private static final int[] T8_4 = new int[] {
+    0xCCB0A91F, 0x740CCE7A, 0x66B96194, 0xDE0506F1,
+    /* T8_4 */
     0x00000000, 0x3D6029B0, 0x7AC05360, 0x47A07AD0, 
     0xF580A6C0, 0xC8E08F70, 0x8F40F5A0, 0xB220DC10, 
     0x30704BC1, 0x0D106271, 0x4AB018A1, 0x77D03111, 
@@ -418,9 +440,8 @@ public class PureJavaCrc32 implements Ch
     0x4834505D, 0x755479ED, 0x32F4033D, 0x0F942A8D, 
     0xBDB4F69D, 0x80D4DF2D, 0xC774A5FD, 0xFA148C4D, 
     0x78441B9C, 0x4524322C, 0x028448FC, 0x3FE4614C, 
-    0x8DC4BD5C, 0xB0A494EC, 0xF704EE3C, 0xCA64C78C
-  };
-  private static final int[] T8_5 = new int[] {
+    0x8DC4BD5C, 0xB0A494EC, 0xF704EE3C, 0xCA64C78C,
+    /* T8_5 */
     0x00000000, 0xCB5CD3A5, 0x4DC8A10B, 0x869472AE, 
     0x9B914216, 0x50CD91B3, 0xD659E31D, 0x1D0530B8, 
     0xEC53826D, 0x270F51C8, 0xA19B2366, 0x6AC7F0C3, 
@@ -484,9 +505,8 @@ public class PureJavaCrc32 implements Ch
     0x15921919, 0xDECECABC, 0x585AB812, 0x93066BB7, 
     0x8E035B0F, 0x455F88AA, 0xC3CBFA04, 0x089729A1, 
     0xF9C19B74, 0x329D48D1, 0xB4093A7F, 0x7F55E9DA, 
-    0x6250D962, 0xA90C0AC7, 0x2F987869, 0xE4C4ABCC
-  };
-  private static final int[] T8_6 = new int[] {
+    0x6250D962, 0xA90C0AC7, 0x2F987869, 0xE4C4ABCC,
+    /* T8_6 */
     0x00000000, 0xA6770BB4, 0x979F1129, 0x31E81A9D, 
     0xF44F2413, 0x52382FA7, 0x63D0353A, 0xC5A73E8E, 
     0x33EF4E67, 0x959845D3, 0xA4705F4E, 0x020754FA, 
@@ -550,9 +570,8 @@ public class PureJavaCrc32 implements Ch
     0x647E3AD9, 0xC209316D, 0xF3E12BF0, 0x55962044, 
     0x90311ECA, 0x3646157E, 0x07AE0FE3, 0xA1D90457, 
     0x579174BE, 0xF1E67F0A, 0xC00E6597, 0x66796E23, 
-    0xA3DE50AD, 0x05A95B19, 0x34414184, 0x92364A30
-  };
-  private static final int[] T8_7 = new int[] {
+    0xA3DE50AD, 0x05A95B19, 0x34414184, 0x92364A30,
+    /* T8_7 */
     0x00000000, 0xCCAA009E, 0x4225077D, 0x8E8F07E3, 
     0x844A0EFA, 0x48E00E64, 0xC66F0987, 0x0AC50919, 
     0xD3E51BB5, 0x1F4F1B2B, 0x91C01CC8, 0x5D6A1C56, 

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java Mon Oct 22 20:43:16 2012
@@ -56,22 +56,38 @@ public class PureJavaCrc32C implements C
   @Override
   public void update(byte[] b, int off, int len) {
     int localCrc = crc;
+
     while(len > 7) {
-      int c0 = b[off++] ^ localCrc;
-      int c1 = b[off++] ^ (localCrc >>>= 8);
-      int c2 = b[off++] ^ (localCrc >>>= 8);
-      int c3 = b[off++] ^ (localCrc >>>= 8);
-      localCrc = (T8_7[c0 & 0xff] ^ T8_6[c1 & 0xff])
-          ^ (T8_5[c2 & 0xff] ^ T8_4[c3 & 0xff]);
+      final int c0 =(b[off+0] ^ localCrc) & 0xff;
+      final int c1 =(b[off+1] ^ (localCrc >>>= 8)) & 0xff;
+      final int c2 =(b[off+2] ^ (localCrc >>>= 8)) & 0xff;
+      final int c3 =(b[off+3] ^ (localCrc >>>= 8)) & 0xff;
+      localCrc = (T[T8_7_start + c0] ^ T[T8_6_start + c1])
+          ^ (T[T8_5_start + c2] ^ T[T8_4_start + c3]);
+
+      final int c4 = b[off+4] & 0xff;
+      final int c5 = b[off+5] & 0xff;
+      final int c6 = b[off+6] & 0xff;
+      final int c7 = b[off+7] & 0xff;
 
-      localCrc ^= (T8_3[b[off++] & 0xff] ^ T8_2[b[off++] & 0xff])
-           ^ (T8_1[b[off++] & 0xff] ^ T8_0[b[off++] & 0xff]);
+      localCrc ^= (T[T8_3_start + c4] ^ T[T8_2_start + c5])
+           ^ (T[T8_1_start + c6] ^ T[T8_0_start + c7]);
 
+      off += 8;
       len -= 8;
     }
-    while(len > 0) {
-      localCrc = (localCrc >>> 8) ^ T8_0[(localCrc ^ b[off++]) & 0xff];
-      len--;
+
+    /* loop unroll - duff's device style */
+    switch(len) {
+      case 7: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 6: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 5: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 4: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 3: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 2: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 1: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      default:
+        /* nothing */
     }
     
     // Publish crc out to object
@@ -80,14 +96,24 @@ public class PureJavaCrc32C implements C
 
   @Override
   final public void update(int b) {
-    crc = (crc >>> 8) ^ T8_0[(crc ^ b) & 0xff];
+    crc = (crc >>> 8) ^ T[T8_0_start + ((crc ^ b) & 0xff)];
   }
     
   // CRC polynomial tables generated by:
   // java -cp build/test/classes/:build/classes/ \
   //   org.apache.hadoop.util.TestPureJavaCrc32\$Table 82F63B78
 
-  static final int[] T8_0 = new int[] {
+  private static final int T8_0_start = 0*256;
+  private static final int T8_1_start = 1*256;
+  private static final int T8_2_start = 2*256;
+  private static final int T8_3_start = 3*256;
+  private static final int T8_4_start = 4*256;
+  private static final int T8_5_start = 5*256;
+  private static final int T8_6_start = 6*256;
+  private static final int T8_7_start = 7*256;
+
+  private static final int[] T = new int[] {
+    /* T8_0 */
     0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, 
     0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB, 
     0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, 
@@ -151,9 +177,8 @@ public class PureJavaCrc32C implements C
     0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81, 
     0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E, 
     0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, 
-    0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351
-  };
-  static final int[] T8_1 = new int[] {
+    0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351, 
+    /* T8_1 */
     0x00000000, 0x13A29877, 0x274530EE, 0x34E7A899, 
     0x4E8A61DC, 0x5D28F9AB, 0x69CF5132, 0x7A6DC945, 
     0x9D14C3B8, 0x8EB65BCF, 0xBA51F356, 0xA9F36B21, 
@@ -217,9 +242,8 @@ public class PureJavaCrc32C implements C
     0x449A2E7E, 0x5738B609, 0x63DF1E90, 0x707D86E7, 
     0x0A104FA2, 0x19B2D7D5, 0x2D557F4C, 0x3EF7E73B, 
     0xD98EEDC6, 0xCA2C75B1, 0xFECBDD28, 0xED69455F, 
-    0x97048C1A, 0x84A6146D, 0xB041BCF4, 0xA3E32483
-  };
-  static final int[] T8_2 = new int[] {
+    0x97048C1A, 0x84A6146D, 0xB041BCF4, 0xA3E32483, 
+    /* T8_2 */
     0x00000000, 0xA541927E, 0x4F6F520D, 0xEA2EC073, 
     0x9EDEA41A, 0x3B9F3664, 0xD1B1F617, 0x74F06469, 
     0x38513EC5, 0x9D10ACBB, 0x773E6CC8, 0xD27FFEB6, 
@@ -283,9 +307,8 @@ public class PureJavaCrc32C implements C
     0xDDA47104, 0x78E5E37A, 0x92CB2309, 0x378AB177, 
     0x437AD51E, 0xE63B4760, 0x0C158713, 0xA954156D, 
     0xE5F54FC1, 0x40B4DDBF, 0xAA9A1DCC, 0x0FDB8FB2, 
-    0x7B2BEBDB, 0xDE6A79A5, 0x3444B9D6, 0x91052BA8
-  };
-  static final int[] T8_3 = new int[] {
+    0x7B2BEBDB, 0xDE6A79A5, 0x3444B9D6, 0x91052BA8, 
+    /* T8_3 */
     0x00000000, 0xDD45AAB8, 0xBF672381, 0x62228939, 
     0x7B2231F3, 0xA6679B4B, 0xC4451272, 0x1900B8CA, 
     0xF64463E6, 0x2B01C95E, 0x49234067, 0x9466EADF, 
@@ -349,9 +372,8 @@ public class PureJavaCrc32C implements C
     0xC747336E, 0x1A0299D6, 0x782010EF, 0xA565BA57, 
     0xBC65029D, 0x6120A825, 0x0302211C, 0xDE478BA4, 
     0x31035088, 0xEC46FA30, 0x8E647309, 0x5321D9B1, 
-    0x4A21617B, 0x9764CBC3, 0xF54642FA, 0x2803E842
-  };
-  static final int[] T8_4 = new int[] {
+    0x4A21617B, 0x9764CBC3, 0xF54642FA, 0x2803E842, 
+    /* T8_4 */
     0x00000000, 0x38116FAC, 0x7022DF58, 0x4833B0F4, 
     0xE045BEB0, 0xD854D11C, 0x906761E8, 0xA8760E44, 
     0xC5670B91, 0xFD76643D, 0xB545D4C9, 0x8D54BB65, 
@@ -415,9 +437,8 @@ public class PureJavaCrc32C implements C
     0xCD796B76, 0xF56804DA, 0xBD5BB42E, 0x854ADB82, 
     0x2D3CD5C6, 0x152DBA6A, 0x5D1E0A9E, 0x650F6532, 
     0x081E60E7, 0x300F0F4B, 0x783CBFBF, 0x402DD013, 
-    0xE85BDE57, 0xD04AB1FB, 0x9879010F, 0xA0686EA3
-  };
-  static final int[] T8_5 = new int[] {
+    0xE85BDE57, 0xD04AB1FB, 0x9879010F, 0xA0686EA3, 
+    /* T8_5 */
     0x00000000, 0xEF306B19, 0xDB8CA0C3, 0x34BCCBDA, 
     0xB2F53777, 0x5DC55C6E, 0x697997B4, 0x8649FCAD, 
     0x6006181F, 0x8F367306, 0xBB8AB8DC, 0x54BAD3C5, 
@@ -481,9 +502,8 @@ public class PureJavaCrc32C implements C
     0x57F4CA8E, 0xB8C4A197, 0x8C786A4D, 0x63480154, 
     0xE501FDF9, 0x0A3196E0, 0x3E8D5D3A, 0xD1BD3623, 
     0x37F2D291, 0xD8C2B988, 0xEC7E7252, 0x034E194B, 
-    0x8507E5E6, 0x6A378EFF, 0x5E8B4525, 0xB1BB2E3C
-  };
-  static final int[] T8_6 = new int[] {
+    0x8507E5E6, 0x6A378EFF, 0x5E8B4525, 0xB1BB2E3C, 
+    /* T8_6 */
     0x00000000, 0x68032CC8, 0xD0065990, 0xB8057558, 
     0xA5E0C5D1, 0xCDE3E919, 0x75E69C41, 0x1DE5B089, 
     0x4E2DFD53, 0x262ED19B, 0x9E2BA4C3, 0xF628880B, 
@@ -547,9 +567,8 @@ public class PureJavaCrc32C implements C
     0x2ED97095, 0x46DA5C5D, 0xFEDF2905, 0x96DC05CD, 
     0x8B39B544, 0xE33A998C, 0x5B3FECD4, 0x333CC01C, 
     0x60F48DC6, 0x08F7A10E, 0xB0F2D456, 0xD8F1F89E, 
-    0xC5144817, 0xAD1764DF, 0x15121187, 0x7D113D4F
-  };
-  static final int[] T8_7 = new int[] {
+    0xC5144817, 0xAD1764DF, 0x15121187, 0x7D113D4F, 
+    /* T8_7 */
     0x00000000, 0x493C7D27, 0x9278FA4E, 0xDB448769, 
     0x211D826D, 0x6821FF4A, 0xB3657823, 0xFA590504, 
     0x423B04DA, 0x0B0779FD, 0xD043FE94, 0x997F83B3, 
@@ -613,6 +632,6 @@ public class PureJavaCrc32C implements C
     0xA777317B, 0xEE4B4C5C, 0x350FCB35, 0x7C33B612, 
     0x866AB316, 0xCF56CE31, 0x14124958, 0x5D2E347F, 
     0xE54C35A1, 0xAC704886, 0x7734CFEF, 0x3E08B2C8, 
-    0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5
+    0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5 
   };
 }

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java Mon Oct 22 20:43:16 2012
@@ -126,6 +126,10 @@ public class RunJar {
     int firstArg = 0;
     String fileName = args[firstArg++];
     File file = new File(fileName);
+    if (!file.exists() || !file.isFile()) {
+      System.err.println("Not a valid JAR: " + file.getCanonicalPath());
+      System.exit(-1);
+    }
     String mainClassName = null;
 
     JarFile jarFile;

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java Mon Oct 22 20:43:16 2012
@@ -608,7 +608,8 @@ public class StringUtils {
             "  build = " + VersionInfo.getUrl() + " -r "
                          + VersionInfo.getRevision()  
                          + "; compiled by '" + VersionInfo.getUser()
-                         + "' on " + VersionInfo.getDate()}
+                         + "' on " + VersionInfo.getDate(),
+            "  java = " + System.getProperty("java.version") }
         )
       );
 

Propchange: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1397381-1401062

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java Mon Oct 22 20:43:16 2012
@@ -472,7 +472,9 @@ public class TestFileUtil {
     setupDirs();
 
     long du = FileUtil.getDU(TEST_DIR);
-    //Only two files (in partitioned) have 4 bytes each
-    Assert.assertEquals(du, 8);
+    // Only two files (in partitioned).  Each has 3 characters + system-specific
+    // line separator.
+    long expected = 2 * (3 + System.getProperty("line.separator").length());
+    Assert.assertEquals(expected, du);
   }
 }

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java Mon Oct 22 20:43:16 2012
@@ -730,6 +730,55 @@ public class TestCodec {
     }
   }
 
+  @Test
+  public void testGzipLongOverflow() throws IOException {
+    LOG.info("testGzipLongOverflow");
+
+    // Don't use native libs for this test.
+    Configuration conf = new Configuration();
+    conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY, false);
+    assertFalse("ZlibFactory is using native libs against request",
+        ZlibFactory.isNativeZlibLoaded(conf));
+
+    // Ensure that the CodecPool has a BuiltInZlibInflater in it.
+    Decompressor zlibDecompressor = ZlibFactory.getZlibDecompressor(conf);
+    assertNotNull("zlibDecompressor is null!", zlibDecompressor);
+    assertTrue("ZlibFactory returned unexpected inflator",
+        zlibDecompressor instanceof BuiltInZlibInflater);
+    CodecPool.returnDecompressor(zlibDecompressor);
+
+    // Now create a GZip text file.
+    String tmpDir = System.getProperty("test.build.data", "/tmp/");
+    Path f = new Path(new Path(tmpDir), "testGzipLongOverflow.bin.gz");
+    BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(
+      new GZIPOutputStream(new FileOutputStream(f.toString()))));
+
+    final int NBUF = 1024 * 4 + 1;
+    final char[] buf = new char[1024 * 1024];
+    for (int i = 0; i < buf.length; i++) buf[i] = '\0';
+    for (int i = 0; i < NBUF; i++) {
+      bw.write(buf);
+    }
+    bw.close();
+
+    // Now read it back, using the CodecPool to establish the
+    // decompressor to use.
+    CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
+    CompressionCodec codec = ccf.getCodec(f);
+    Decompressor decompressor = CodecPool.getDecompressor(codec);
+    FileSystem fs = FileSystem.getLocal(conf);
+    InputStream is = fs.open(f);
+    is = codec.createInputStream(is, decompressor);
+    BufferedReader br = new BufferedReader(new InputStreamReader(is));
+    for (int j = 0; j < NBUF; j++) {
+      int n = br.read(buf);
+      assertEquals("got wrong read length!", n, buf.length);
+      for (int i = 0; i < buf.length; i++)
+        assertEquals("got wrong byte!", buf[i], '\0');
+    }
+    br.close();
+  }
+
   public void testGzipCodecWrite(boolean useNative) throws IOException {
     // Create a gzipped file using a compressor from the CodecPool,
     // and try to read it back via the regular GZIPInputStream.

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestVLong.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestVLong.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestVLong.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestVLong.java Mon Oct 22 20:43:16 2012
@@ -141,7 +141,7 @@ public class TestVLong extends TestCase 
       int shift = rng.nextInt(Long.SIZE) + 1;
       long mask = (1L << shift) - 1;
       long a = ((long) rng.nextInt()) << 32;
-      long b = ((long) rng.nextInt()) & 0xffffffff;
+      long b = ((long) rng.nextInt()) & 0xffffffffL;
       data[i] = (a + b) & mask;
     }
     

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java Mon Oct 22 20:43:16 2012
@@ -19,10 +19,7 @@
 package org.apache.hadoop.ipc;
 
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
+import static org.junit.Assert.*;
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
@@ -44,6 +41,7 @@ import org.apache.hadoop.fs.CommonConfig
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ipc.Client.ConnectionId;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.KerberosInfo;
 import org.apache.hadoop.security.SaslInputStream;
 import org.apache.hadoop.security.SaslRpcClient;
@@ -449,23 +447,100 @@ public class TestSaslRPC {
     }
     System.out.println("Test is successful.");
   }
-  
+
+  // insecure -> insecure
+  @Test
+  public void testInsecureClientInsecureServer() throws Exception {
+    assertEquals(AuthenticationMethod.SIMPLE,
+                 getAuthMethod(false, false, false));
+  }
+
+  @Test
+  public void testInsecureClientInsecureServerWithToken() throws Exception {
+    assertEquals(AuthenticationMethod.TOKEN,
+                 getAuthMethod(false, false, true));
+  }
+
+  // insecure -> secure
+  @Test
+  public void testInsecureClientSecureServer() throws Exception {
+    RemoteException e = null;
+    try {
+      getAuthMethod(false, true, false);
+    } catch (RemoteException re) {
+      e = re;
+    }
+    assertNotNull(e);
+    assertEquals(AccessControlException.class.getName(), e.getClassName());
+  }
+
+  @Test
+  public void testInsecureClientSecureServerWithToken() throws Exception {
+    assertEquals(AuthenticationMethod.TOKEN,
+                 getAuthMethod(false, true, true));
+  }
+
+  // secure -> secure
+  @Test
+  public void testSecureClientSecureServer() throws Exception {
+    /* Should be this when multiple secure auths are supported and we can
+     * dummy one out:
+     *     assertEquals(AuthenticationMethod.SECURE_AUTH_METHOD,
+     *                  getAuthMethod(true, true, false));
+     */
+    try {
+      getAuthMethod(true, true, false);
+    } catch (IOException ioe) {
+      // can't actually test kerberos w/o kerberos...
+      String expectedError = "Failed to specify server's Kerberos principal";
+      String actualError = ioe.getMessage();
+      assertTrue("["+actualError+"] doesn't start with ["+expectedError+"]",
+          actualError.contains(expectedError));
+    }
+  }
+
   @Test
-  public void testDigestAuthMethodSecureServer() throws Exception {
-    checkDigestAuthMethod(true);
+  public void testSecureClientSecureServerWithToken() throws Exception {
+    assertEquals(AuthenticationMethod.TOKEN,
+                 getAuthMethod(true, true, true));
   }
 
+  // secure -> insecure
   @Test
-  public void testDigestAuthMethodInsecureServer() throws Exception {
-    checkDigestAuthMethod(false);
+  public void testSecureClientInsecureServerWithToken() throws Exception {
+    assertEquals(AuthenticationMethod.TOKEN,
+                 getAuthMethod(true, false, true));
   }
 
-  private void checkDigestAuthMethod(boolean secureServer) throws Exception {
+  @Test
+  public void testSecureClientInsecureServer() throws Exception {
+    /* Should be this when multiple secure auths are supported and we can
+     * dummy one out:
+     *     assertEquals(AuthenticationMethod.SIMPLE
+     *                  getAuthMethod(true, false, false));
+     */
+    try {
+      getAuthMethod(true, false, false);
+    } catch (IOException ioe) {
+      // can't actually test kerberos w/o kerberos...
+      String expectedError = "Failed to specify server's Kerberos principal";
+      String actualError = ioe.getMessage();
+      assertTrue("["+actualError+"] doesn't start with ["+expectedError+"]",
+          actualError.contains(expectedError));
+    }
+  }
+
+
+  private AuthenticationMethod getAuthMethod(final boolean isSecureClient,
+                                             final boolean isSecureServer,
+                                             final boolean useToken
+                                             
+      ) throws Exception {
     TestTokenSecretManager sm = new TestTokenSecretManager();
     Server server = new RPC.Builder(conf).setProtocol(TestSaslProtocol.class)
         .setInstance(new TestSaslImpl()).setBindAddress(ADDRESS).setPort(0)
         .setNumHandlers(5).setVerbose(true).setSecretManager(sm).build();      
-    if (secureServer) {
+    if (isSecureServer) {
       server.enableSecurity();
     } else {
       server.disableSecurity();
@@ -474,30 +549,36 @@ public class TestSaslRPC {
 
     final UserGroupInformation current = UserGroupInformation.getCurrentUser();
     final InetSocketAddress addr = NetUtils.getConnectAddress(server);
-    TestTokenIdentifier tokenId = new TestTokenIdentifier(new Text(current
-        .getUserName()));
-    Token<TestTokenIdentifier> token = new Token<TestTokenIdentifier>(tokenId,
-        sm);
-    SecurityUtil.setTokenService(token, addr);
-    current.addToken(token);
+    if (useToken) {
+      TestTokenIdentifier tokenId = new TestTokenIdentifier(
+          new Text(current.getUserName()));
+      Token<TestTokenIdentifier> token =
+          new Token<TestTokenIdentifier>(tokenId, sm);
+      SecurityUtil.setTokenService(token, addr);
+      current.addToken(token);
+    }
 
-    current.doAs(new PrivilegedExceptionAction<Object>() {
-      @Override
-      public Object run() throws IOException {
-        TestSaslProtocol proxy = null;
-        try {
-          proxy = (TestSaslProtocol) RPC.getProxy(TestSaslProtocol.class,
-              TestSaslProtocol.versionID, addr, conf);
-          Assert.assertEquals(AuthenticationMethod.TOKEN, proxy.getAuthMethod());
-        } finally {
-          if (proxy != null) {
-            RPC.stopProxy(proxy);
+    conf.set(HADOOP_SECURITY_AUTHENTICATION, isSecureClient ? "kerberos" : "simple");
+    UserGroupInformation.setConfiguration(conf);
+    try {
+      return current.doAs(new PrivilegedExceptionAction<AuthenticationMethod>() {
+        @Override
+        public AuthenticationMethod run() throws IOException {
+          TestSaslProtocol proxy = null;
+          try {
+            proxy = (TestSaslProtocol) RPC.getProxy(TestSaslProtocol.class,
+                TestSaslProtocol.versionID, addr, conf);
+            return proxy.getAuthMethod();
+          } finally {
+            if (proxy != null) {
+              RPC.stopProxy(proxy);
+            }
           }
         }
-        return null;
-      }
-    });
-    server.stop();
+      });
+    } finally {
+      server.stop();
+    }
   }
   
   public static void main(String[] args) throws Exception {

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java Mon Oct 22 20:43:16 2012
@@ -78,5 +78,29 @@ public class TestJMXJsonServlet extends 
         "/jmx?get=java.lang:type=Memory::"));
     LOG.info("/jmx RESULT: "+result);
     assertReFind("\"ERROR\"", result);
+
+    // test to get JSONP result
+    result = readOutput(new URL(baseUrl, "/jmx?qry=java.lang:type=Memory&callback=mycallback1"));
+    LOG.info("/jmx?qry=java.lang:type=Memory&callback=mycallback RESULT: "+result);
+    assertReFind("^mycallback1\\(\\{", result);
+    assertReFind("\\}\\);$", result);
+
+    // negative test to get an attribute of a mbean as JSONP
+    result = readOutput(new URL(baseUrl,
+        "/jmx?get=java.lang:type=Memory::&callback=mycallback2"));
+    LOG.info("/jmx RESULT: "+result);
+    assertReFind("^mycallback2\\(\\{", result);
+    assertReFind("\"ERROR\"", result);
+    assertReFind("\\}\\);$", result);
+
+    // test to get an attribute of a mbean as JSONP
+    result = readOutput(new URL(baseUrl,
+        "/jmx?get=java.lang:type=Memory::HeapMemoryUsage&callback=mycallback3"));
+    LOG.info("/jmx RESULT: "+result);
+    assertReFind("^mycallback3\\(\\{", result);
+    assertReFind("\"name\"\\s*:\\s*\"java.lang:type=Memory\"", result);
+    assertReFind("\"committed\"\\s*:", result);
+    assertReFind("\\}\\);$", result);
+
   }
 }

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/util/TestSampleQuantiles.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/util/TestSampleQuantiles.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/util/TestSampleQuantiles.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/util/TestSampleQuantiles.java Mon Oct 22 20:43:16 2012
@@ -18,9 +18,7 @@
 
 package org.apache.hadoop.metrics2.util;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import static org.junit.Assert.*;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -28,7 +26,6 @@ import java.util.Collections;
 import java.util.Map;
 import java.util.Random;
 
-import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -54,18 +51,22 @@ public class TestSampleQuantiles {
     // Counts start off zero
     assertEquals(estimator.getCount(), 0);
     assertEquals(estimator.getSampleCount(), 0);
-    try {
-      estimator.snapshot();
-      fail("Expected IOException from empty window");
-    } catch (IOException e) {
-      GenericTestUtils.assertExceptionContains("No samples", e);
-    }
+    
+    // Snapshot should be null if there are no entries.
+    assertNull(estimator.snapshot());
 
     // Count increment correctly by 1
     estimator.insert(1337);
     assertEquals(estimator.getCount(), 1);
     estimator.snapshot();
     assertEquals(estimator.getSampleCount(), 1);
+    
+    assertEquals(
+        "50.00 %ile +/- 5.00%: 1337\n" +
+        "75.00 %ile +/- 2.50%: 1337\n" +
+        "90.00 %ile +/- 1.00%: 1337\n" +
+        "95.00 %ile +/- 0.50%: 1337\n" +
+        "99.00 %ile +/- 0.10%: 1337", estimator.toString());
   }
 
   /**
@@ -80,12 +81,7 @@ public class TestSampleQuantiles {
     estimator.clear();
     assertEquals(estimator.getCount(), 0);
     assertEquals(estimator.getSampleCount(), 0);
-    try {
-      estimator.snapshot();
-      fail("Expected IOException for an empty window.");
-    } catch (IOException e) {
-      GenericTestUtils.assertExceptionContains("No samples", e);
-    }
+    assertNull(estimator.snapshot());
   }
 
   /**

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPureJavaCrc32.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPureJavaCrc32.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPureJavaCrc32.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPureJavaCrc32.java Mon Oct 22 20:43:16 2012
@@ -142,16 +142,14 @@ public class TestPureJavaCrc32 {
       for (int j = 0; j < tables.length; j++) {
         final int[] t = tables[j];
         final StringBuilder b = new StringBuilder();
-        b.append(String.format("  static final int[] " + nameformat
-            + " = new int[] {", j));
+        b.append(String.format("    /* "+ nameformat +" */", j));
         for (int i = 0; i < t.length;) {
           b.append("\n    ");
           for(int k = 0; k < 4; k++) {
             b.append(String.format("0x%08X, ", t[i++]));
           }
         }
-        b.setCharAt(b.length() - 2, '\n');
-        s[j] = b.toString() + " };\n";
+        s[j] = b.toString();
       }
       return s;
     }
@@ -159,10 +157,23 @@ public class TestPureJavaCrc32 {
     @Override
     public String toString() {
       final StringBuilder b = new StringBuilder();
-      for(String s : toStrings(String.format("T%d_",
-          Integer.numberOfTrailingZeros(tables[0].length)) + "%d")) {
+
+      final String tableFormat = String.format("T%d_", 
+        Integer.numberOfTrailingZeros(tables[0].length)) + "%d";
+      final String startFormat = "  private static final int "+tableFormat+"_start = %d*256;";
+
+      for (int j = 0; j < tables.length; j++) {
+        b.append(String.format(startFormat, j, j));
+        b.append("\n");
+      }
+
+      b.append("  private static final int[] T = new int[] {");
+      for(String s : toStrings(tableFormat)) {
+        b.append("\n");
         b.append(s);
       }
+      b.setCharAt(b.length() - 2, '\n');
+      b.append(" };\n");
       return b.toString();
     }
 



Mime
View raw message