hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jeag...@apache.org
Subject svn commit: r1453613 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/shell/Display.java src/test/resources/testConf.xml
Date Wed, 06 Mar 2013 23:17:15 GMT
Author: jeagles
Date: Wed Mar  6 23:17:14 2013
New Revision: 1453613

URL: http://svn.apache.org/r1453613
Log:
HADOOP-9209. Add shell command to dump file checksums (Todd Lipcon via jeagles)

Modified:
    hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1453613&r1=1453612&r2=1453613&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Mar  6 23:17:14
2013
@@ -1506,6 +1506,9 @@ Release 0.23.7 - UNRELEASED
 
     HADOOP-9352. Expose UGI.setLoginUser for tests (daryn)
 
+    HADOOP-9209. Add shell command to dump file checksums (Todd Lipcon via
+    jeagles)
+
   OPTIMIZATIONS
 
     HADOOP-8462. Native-code implementation of bzip2 codec. (Govind Kamat via

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java?rev=1453613&r1=1453612&r2=1453613&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
Wed Mar  6 23:17:14 2013
@@ -36,6 +36,7 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileChecksum;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathIsDirectoryException;
@@ -47,13 +48,14 @@ import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.compress.CompressionCodec;
 import org.apache.hadoop.io.compress.CompressionCodecFactory;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
 import org.codehaus.jackson.JsonEncoding;
 import org.codehaus.jackson.JsonFactory;
 import org.codehaus.jackson.JsonGenerator;
 import org.codehaus.jackson.util.MinimalPrettyPrinter;
 
 /**
- * Display contents of files 
+ * Display contents or checksums of files 
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
@@ -62,6 +64,7 @@ class Display extends FsCommand {
   public static void registerCommands(CommandFactory factory) {
     factory.addClass(Cat.class, "-cat");
     factory.addClass(Text.class, "-text");
+    factory.addClass(Checksum.class, "-checksum");
   }
 
   /**
@@ -161,6 +164,36 @@ class Display extends FsCommand {
       return i;
     }
   }
+  
+  public static class Checksum extends Display {
+    public static final String NAME = "checksum";
+    public static final String USAGE = "<src> ...";
+    public static final String DESCRIPTION =
+      "Dump checksum information for files that match the file\n" +
+      "pattern <src> to stdout. Note that this requires a round-trip\n" +
+      "to a datanode storing each block of the file, and thus is not\n" +
+      "efficient to run on a large number of files. The checksum of a\n" +
+      "file depends on its content, block size and the checksum\n" +
+      "algorithm and parameters used for creating the file.";
+
+    @Override
+    protected void processPath(PathData item) throws IOException {
+      if (item.stat.isDirectory()) {
+        throw new PathIsDirectoryException(item.toString());
+      }
+
+      FileChecksum checksum = item.fs.getFileChecksum(item.path);
+      if (checksum == null) {
+        out.printf("%s\tNONE\t\n", item.toString());
+      } else {
+        String checksumString = StringUtils.byteToHexString(
+            checksum.getBytes(), 0, checksum.getLength());
+        out.printf("%s\t%s\t%s\n",
+            item.toString(), checksum.getAlgorithmName(),
+            checksumString);
+      }
+    }
+  }
 
   protected class TextRecordInputStream extends InputStream {
     SequenceFile.Reader r;

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml?rev=1453613&r1=1453612&r2=1453613&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
Wed Mar  6 23:17:14 2013
@@ -487,6 +487,22 @@
     </test>
 
     <test> <!-- TESTED -->
+      <description>help: help for checksum</description>
+      <test-commands>
+        <command>-help checksum</command>
+      </test-commands>
+      <cleanup-commands>
+        <!-- No cleanup -->
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^-checksum &lt;src&gt; \.\.\.:( |\t)*Dump checksum
information for files.*</expected-output>
+        </comparator>
+      </comparators>
+    </test>
+
+    <test> <!-- TESTED -->
       <description>help: help for copyToLocal</description>
       <test-commands>
         <command>-help copyToLocal</command>



Mime
View raw message