hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r1125163 - in /hbase/trunk: ./ src/docbkx/ src/main/java/org/apache/hadoop/hbase/ src/main/java/org/apache/hadoop/hbase/io/hfile/ src/main/java/org/apache/hadoop/hbase/util/ src/test/java/org/apache/hadoop/hbase/io/hfile/ src/test/java/org/...
Date Thu, 19 May 2011 23:00:47 GMT
Author: stack
Date: Thu May 19 23:00:47 2011
New Revision: 1125163

URL: http://svn.apache.org/viewvc?rev=1125163&view=rev
Log:
HBASE-3691 Add compressor support for 'snappy', google's compressor

Modified:
    hbase/trunk/CHANGES.txt
    hbase/trunk/src/docbkx/book.xml
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java

Modified: hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1125163&r1=1125162&r2=1125163&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Thu May 19 23:00:47 2011
@@ -257,6 +257,8 @@ Release 0.91.0 - Unreleased
    HBASE-3837  Show regions in transition on the master web page (todd)
    HBASE-3839  Add monitoring of currently running tasks to the master and
                RS web UIs
+   HBASE-3691  Add compressor support for 'snappy', google's compressor
+               (Nichole Treadway and Nicholas Telford)
 
 Release 0.90.4 - Unreleased
 

Modified: hbase/trunk/src/docbkx/book.xml
URL: http://svn.apache.org/viewvc/hbase/trunk/src/docbkx/book.xml?rev=1125163&r1=1125162&r2=1125163&view=diff
==============================================================================
--- hbase/trunk/src/docbkx/book.xml (original)
+++ hbase/trunk/src/docbkx/book.xml Thu May 19 23:00:47 2011
@@ -1373,6 +1373,45 @@ false
     reports in your logs; see <xref linkend="brand.new.compressor" />).
     </para>
     </section>
+    <section xml:id="snappy.compression">
+    <title>
+    SNAPPY
+    </title>
+    <para>
+        To set snappy compression on a column family, do as following:
+        <orderedlist>
+            <listitem>
+                <para>
+                 Install hadoop-snappy using these instructions: http://code.google.com/p/hadoop-snappy/
+                </para>
+            </listitem>
+            <listitem>
+                <para>
+                    You need to ensure the hadoop-snappy libs (incl. the native libs) are
in the HBase classpath. One way to do this is
+                    to just symlink the libs from <filename>HADOOP_HOME/lib</filename>
to <filename>HBASE_HOME/lib</filename>.
+                    This needs to be done on all HBase nodes, as with LZO.
+                </para>
+            </listitem>
+            <listitem>
+                <para>
+        Use CompressionTest to verify snappy support is enabled and the libs can be loaded:
+        <programlisting>$ hbase org.apache.hadoop.hbase.util.CompressionTest hdfs://host/path/to/hbase
snappy</programlisting>
+                </para>
+            </listitem>
+            <listitem>
+                <para>
+        Create a column family with snappy compression and verify it in the hbase shell:
+        <programlisting>$ hbase> create 't1', { NAME => 'cf1', COMPRESSION =>
'snappy' }
+hbase> describe 't1'</programlisting>
+        In the output of the "describe" command, you need to ensure it lists "COMPRESSION
=> 'snappy'"
+                </para>
+            </listitem>
+
+        </orderedlist>
+
+    </para>
+    </section>
+
   </appendix>
 
   <appendix xml:id="faq">

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java?rev=1125163&r1=1125162&r2=1125163&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java Thu May 19 23:00:47
2011
@@ -432,6 +432,7 @@ public class HColumnDescriptor implement
     switch (type) {
       case LZO: compressionType = "LZO"; break;
       case GZ: compressionType = "GZ"; break;
+      case SNAPPY: compressionType = "SNAPPY"; break;
       default: compressionType = "NONE"; break;
     }
     setValue(COMPRESSION, compressionType);
@@ -456,6 +457,7 @@ public class HColumnDescriptor implement
     switch (type) {
       case LZO: compressionType = "LZO"; break;
       case GZ: compressionType = "GZ"; break;
+      case SNAPPY: compressionType = "SNAPPY"; break;
       default: compressionType = "NONE"; break;
     }
     setValue(COMPRESSION_COMPACT, compressionType);

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java?rev=1125163&r1=1125162&r2=1125163&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java Thu May 19
23:00:47 2011
@@ -140,6 +140,25 @@ public final class Compression {
 
         return downStream;
       }
+    },
+    SNAPPY("snappy") {
+        // Use base type to avoid compile-time dependencies.
+        private transient CompressionCodec snappyCodec;
+
+        @Override
+        CompressionCodec getCodec(Configuration conf) {
+          if (snappyCodec == null) {
+            try {
+              Class<?> externalCodec =
+                  ClassLoader.getSystemClassLoader().loadClass("org.apache.hadoop.io.compress.SnappyCodec");
+              snappyCodec = (CompressionCodec) ReflectionUtils.newInstance(externalCodec,

+                  conf);
+            } catch (ClassNotFoundException e) {
+              throw new RuntimeException(e);
+            }
+          }
+          return snappyCodec;
+        }
     };
 
     private final Configuration conf;

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java?rev=1125163&r1=1125162&r2=1125163&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java Thu May 19
23:00:47 2011
@@ -94,7 +94,7 @@ public class CompressionTest {
 
   public static void usage() {
     System.err.println(
-      "Usage: CompressionTest <path> none|gz|lzo\n" +
+      "Usage: CompressionTest <path> none|gz|lzo|snappy\n" +
       "\n" +
       "For example:\n" +
       "  hbase " + CompressionTest.class + " file:///tmp/testfile gz\n");

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java?rev=1125163&r1=1125162&r2=1125163&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java Thu
May 19 23:00:47 2011
@@ -129,7 +129,7 @@ public class TestHFilePerformance extend
    * @param fileType "HFile" or "SequenceFile"
    * @param keyLength
    * @param valueLength
-   * @param codecName "none", "lzo", "gz"
+   * @param codecName "none", "lzo", "gz", "snappy"
    * @param rows number of rows to be written.
    * @param writeMethod used for HFile only.
    * @param minBlockSize used for HFile only.

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java?rev=1125163&r1=1125162&r2=1125163&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java Thu May
19 23:00:47 2011
@@ -281,7 +281,7 @@ public class TestHFileSeek extends TestC
 
     private Options buildOptions() {
       Option compress =
-          OptionBuilder.withLongOpt("compress").withArgName("[none|lzo|gz]")
+          OptionBuilder.withLongOpt("compress").withArgName("[none|lzo|gz|snappy]")
               .hasArg().withDescription("compression scheme").create('c');
 
       Option fileSize =
@@ -446,7 +446,7 @@ public class TestHFileSeek extends TestC
 
     private void validateOptions() throws ParseException {
       if (!compress.equals("none") && !compress.equals("lzo")
-          && !compress.equals("gz")) {
+          && !compress.equals("gz") && !compress.equals("snappy")) {
         throw new ParseException("Unknown compression scheme: " + compress);
       }
 

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java?rev=1125163&r1=1125162&r2=1125163&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
(original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
Thu May 19 23:00:47 2011
@@ -438,6 +438,9 @@ public class TestHFileOutputFormat  {
       familyToCompression.put("Family1!@#!@#&", Compression.Algorithm.LZO);
     }
     if (numCfs-- > 0) {
+      familyToCompression.put("Family2=asdads&!AASD", Compression.Algorithm.SNAPPY);
+    }
+    if (numCfs-- > 0) {
       familyToCompression.put("Family2=asdads&!AASD", Compression.Algorithm.GZ);
     }
     if (numCfs-- > 0) {

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java?rev=1125163&r1=1125162&r2=1125163&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java Thu May
19 23:00:47 2011
@@ -54,5 +54,6 @@ public class TestCompressionTest {
     assertFalse(CompressionTest.testCompression("LZO"));
     assertTrue(CompressionTest.testCompression("NONE"));
     assertTrue(CompressionTest.testCompression("GZ"));
+    assertTrue(CompressionTest.testCompression("SNAPPY"));
   }
 }



Mime
View raw message