hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r555768 - in /lucene/hadoop/trunk: ./ conf/ src/java/org/apache/hadoop/io/ src/java/org/apache/hadoop/io/compress/ src/java/org/apache/hadoop/io/compress/zlib/ src/java/org/apache/hadoop/util/ src/test/org/apache/hadoop/io/ src/test/org/apa...
Date Thu, 12 Jul 2007 21:08:23 GMT
Author: cutting
Date: Thu Jul 12 14:08:22 2007
New Revision: 555768

URL: http://svn.apache.org/viewvc?view=rev&rev=555768
Log:
HADOOP-1570.  Permit jobs to enable and disable use of Hadoop's native library.  Contributed
by Arun.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/conf/hadoop-default.xml
    lucene/hadoop/trunk/src/java/org/apache/hadoop/io/SequenceFile.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/DefaultCodec.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/GzipCodec.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/LzoCodec.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/util/NativeCodeLoader.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/io/TestSequenceFile.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/io/compress/TestCodec.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=555768&r1=555767&r2=555768
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Thu Jul 12 14:08:22 2007
@@ -343,6 +343,9 @@
 106. HADOOP-1547.  Provide examples for aggregate library.
      (Runping Qi via tomwhite)
 
+107. HADOOP-1570.  Permit jobs to enable and disable the use of
+     hadoop's native library.  (Arun C Murthy via cutting)
+
 
 Release 0.13.0 - 2007-06-08
 

Modified: lucene/hadoop/trunk/conf/hadoop-default.xml
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/conf/hadoop-default.xml?view=diff&rev=555768&r1=555767&r2=555768
==============================================================================
--- lucene/hadoop/trunk/conf/hadoop-default.xml (original)
+++ lucene/hadoop/trunk/conf/hadoop-default.xml Thu Jul 12 14:08:22 2007
@@ -15,6 +15,11 @@
   <description>A base for other temporary directories.</description>
 </property>
 
+<property>
+  <name>hadoop.native.lib</name>
+  <value>true</value>
+  <description>Should native hadoop libraries, if present, be used.</description>
+</property>
 
 <!--- logging properties -->
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/io/SequenceFile.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/io/SequenceFile.java?view=diff&rev=555768&r1=555767&r2=555768
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/io/SequenceFile.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/io/SequenceFile.java Thu Jul 12 14:08:22
2007
@@ -193,7 +193,7 @@
     throws IOException {
     if ((codec instanceof GzipCodec) && 
         !NativeCodeLoader.isNativeCodeLoaded() && 
-        !ZlibFactory.isNativeZlibLoaded()) {
+        !ZlibFactory.isNativeZlibLoaded(conf)) {
       throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                          "GzipCodec without native-hadoop code!");
     }
@@ -234,7 +234,7 @@
                  Progressable progress, Metadata metadata) throws IOException {
     if ((codec instanceof GzipCodec) && 
         !NativeCodeLoader.isNativeCodeLoaded() && 
-        !ZlibFactory.isNativeZlibLoaded()) {
+        !ZlibFactory.isNativeZlibLoaded(conf)) {
       throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                          "GzipCodec without native-hadoop code!");
     }
@@ -295,7 +295,7 @@
     throws IOException {
     if (codec != null && (codec instanceof GzipCodec) && 
         !NativeCodeLoader.isNativeCodeLoaded() && 
-        !ZlibFactory.isNativeZlibLoaded()) {
+        !ZlibFactory.isNativeZlibLoaded(conf)) {
       throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                          "GzipCodec without native-hadoop code!");
     }
@@ -336,7 +336,7 @@
   throws IOException {
   if (codec != null && (codec instanceof GzipCodec) && 
       !NativeCodeLoader.isNativeCodeLoaded() && 
-      !ZlibFactory.isNativeZlibLoaded()) {
+      !ZlibFactory.isNativeZlibLoaded(conf)) {
     throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                        "GzipCodec without native-hadoop code!");
   }
@@ -375,7 +375,7 @@
     throws IOException {
     if ((codec instanceof GzipCodec) && 
         !NativeCodeLoader.isNativeCodeLoaded() && 
-        !ZlibFactory.isNativeZlibLoaded()) {
+        !ZlibFactory.isNativeZlibLoaded(conf)) {
       throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                          "GzipCodec without native-hadoop code!");
     }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/DefaultCodec.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/DefaultCodec.java?view=diff&rev=555768&r1=555767&r2=555768
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/DefaultCodec.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/DefaultCodec.java Thu Jul 12
14:08:22 2007
@@ -52,11 +52,11 @@
   }
 
   public Class getCompressorType() {
-    return ZlibFactory.getZlibCompressorType();
+    return ZlibFactory.getZlibCompressorType(conf);
   }
 
   public Compressor createCompressor() {
-    return ZlibFactory.getZlibCompressor();
+    return ZlibFactory.getZlibCompressor(conf);
   }
 
   public CompressionInputStream createInputStream(InputStream in) 
@@ -73,11 +73,11 @@
   }
 
   public Class getDecompressorType() {
-    return ZlibFactory.getZlibDecompressorType();
+    return ZlibFactory.getZlibDecompressorType(conf);
   }
 
   public Decompressor createDecompressor() {
-    return ZlibFactory.getZlibDecompressor();
+    return ZlibFactory.getZlibDecompressor(conf);
   }
   
   public String getDefaultExtension() {

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/GzipCodec.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/GzipCodec.java?view=diff&rev=555768&r1=555767&r2=555768
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/GzipCodec.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/GzipCodec.java Thu Jul 12 14:08:22
2007
@@ -135,7 +135,7 @@
   
   public CompressionOutputStream createOutputStream(OutputStream out) 
     throws IOException {
-    return (ZlibFactory.isNativeZlibLoaded()) ?
+    return (ZlibFactory.isNativeZlibLoaded(conf)) ?
                new CompressorStream(out, createCompressor(),
                                     conf.getInt("io.file.buffer.size", 4*1024)) :
                new GzipOutputStream(out);
@@ -153,7 +153,7 @@
   }
 
   public Compressor createCompressor() {
-    return (ZlibFactory.isNativeZlibLoaded()) ?
+    return (ZlibFactory.isNativeZlibLoaded(conf)) ?
                new ZlibCompressor(ZlibCompressor.CompressionLevel.DEFAULT_COMPRESSION,
                                   ZlibCompressor.CompressionStrategy.DEFAULT_STRATEGY,
                                   ZlibCompressor.CompressionHeader.GZIP_FORMAT,
@@ -162,12 +162,12 @@
   }
 
   public Class getCompressorType() {
-    return ZlibFactory.getZlibCompressorType();
+    return ZlibFactory.getZlibCompressorType(conf);
   }
 
   public CompressionInputStream createInputStream(InputStream in) 
   throws IOException {
-  return (ZlibFactory.isNativeZlibLoaded()) ?
+  return (ZlibFactory.isNativeZlibLoaded(conf)) ?
              new DecompressorStream(in, createDecompressor(),
                                     conf.getInt("io.file.buffer.size", 
                                                 4*1024)) :
@@ -185,14 +185,14 @@
   }
 
   public Decompressor createDecompressor() {
-    return (ZlibFactory.isNativeZlibLoaded()) ?
+    return (ZlibFactory.isNativeZlibLoaded(conf)) ?
                new ZlibDecompressor(ZlibDecompressor.CompressionHeader.AUTODETECT_GZIP_ZLIB,
                                     64*1024) :
                null;                               
   }
 
   public Class getDecompressorType() {
-    return ZlibFactory.getZlibDecompressorType();
+    return ZlibFactory.getZlibDecompressorType(conf);
   }
 
   public String getDefaultExtension() {

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/LzoCodec.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/LzoCodec.java?view=diff&rev=555768&r1=555767&r2=555768
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/LzoCodec.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/LzoCodec.java Thu Jul 12 14:08:22
2007
@@ -69,17 +69,18 @@
   /**
    * Check if native-lzo library is loaded & initialized.
    * 
+   * @param conf configuration
    * @return <code>true</code> if native-lzo library is loaded & initialized;
    *         else <code>false</code>
    */
-  public static boolean isNativeLzoLoaded() {
-    return nativeLzoLoaded;
+  public static boolean isNativeLzoLoaded(Configuration conf) {
+    return nativeLzoLoaded && conf.getBoolean("hadoop.native.lib", true);
   }
   
   public CompressionOutputStream createOutputStream(OutputStream out) 
     throws IOException {
     // Ensure native-lzo library is loaded & initialized
-    if (!isNativeLzoLoaded()) {
+    if (!isNativeLzoLoaded(conf)) {
       throw new RuntimeException("native-lzo library not available");
     }
     
@@ -128,7 +129,7 @@
                                                     Compressor compressor) 
   throws IOException {
     // Ensure native-lzo library is loaded & initialized
-    if (!isNativeLzoLoaded()) {
+    if (!isNativeLzoLoaded(conf)) {
       throw new RuntimeException("native-lzo library not available");
     }
     
@@ -153,7 +154,7 @@
 
   public Class getCompressorType() {
     // Ensure native-lzo library is loaded & initialized
-    if (!isNativeLzoLoaded()) {
+    if (!isNativeLzoLoaded(conf)) {
       throw new RuntimeException("native-lzo library not available");
     }
     
@@ -162,7 +163,7 @@
 
   public Compressor createCompressor() {
     // Ensure native-lzo library is loaded & initialized
-    if (!isNativeLzoLoaded()) {
+    if (!isNativeLzoLoaded(conf)) {
       throw new RuntimeException("native-lzo library not available");
     }
     
@@ -181,7 +182,7 @@
   public CompressionInputStream createInputStream(InputStream in) 
     throws IOException {
     // Ensure native-lzo library is loaded & initialized
-    if (!isNativeLzoLoaded()) {
+    if (!isNativeLzoLoaded(conf)) {
       throw new IOException("native-lzo library not available");
     }
     
@@ -204,7 +205,7 @@
                                                   Decompressor decompressor) 
   throws IOException {
     // Ensure native-lzo library is loaded & initialized
-    if (!isNativeLzoLoaded()) {
+    if (!isNativeLzoLoaded(conf)) {
       throw new RuntimeException("native-lzo library not available");
     }
     
@@ -215,7 +216,7 @@
 
   public Class getDecompressorType() {
     // Ensure native-lzo library is loaded & initialized
-    if (!isNativeLzoLoaded()) {
+    if (!isNativeLzoLoaded(conf)) {
       throw new RuntimeException("native-lzo library not available");
     }
     
@@ -224,7 +225,7 @@
 
   public Decompressor createDecompressor() {
     // Ensure native-lzo library is loaded & initialized
-    if (!isNativeLzoLoaded()) {
+    if (!isNativeLzoLoaded(conf)) {
       throw new RuntimeException("native-lzo library not available");
     }
     

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java?view=diff&rev=555768&r1=555767&r2=555768
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java Thu Jul
12 14:08:22 2007
@@ -20,10 +20,10 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.compress.Compressor;
 import org.apache.hadoop.io.compress.Decompressor;
 import org.apache.hadoop.util.NativeCodeLoader;
-import org.apache.hadoop.util.StringUtils;
 
 /**
  * A collection of factories to create the right 
@@ -50,54 +50,58 @@
   }
   
   /**
-   * Check if native-zlib code is loaded and initialized correctly.
+   * Check if native-zlib code is loaded & initialized correctly and 
+   * can be loaded for this job.
    * 
-   * @return <code>true</code> if native-zlib is loaded and initialized, 
-   *         else <code>false</code>
+   * @param conf configuration
+   * @return <code>true</code> if native-zlib is loaded & initialized 
+   *         and can be loaded for this job, else <code>false</code>
    */
-  public static boolean isNativeZlibLoaded() {
-    return nativeZlibLoaded; 
+  public static boolean isNativeZlibLoaded(Configuration conf) {
+    return nativeZlibLoaded && conf.getBoolean("hadoop.native.lib", true); 
   }
   
   /**
    * Return the appropriate type of the zlib compressor. 
    * 
+   * @param conf configuration
    * @return the appropriate type of the zlib compressor.
    */
-  public static Class getZlibCompressorType() {
-    return (nativeZlibLoaded) ? 
+  public static Class getZlibCompressorType(Configuration conf) {
+    return (isNativeZlibLoaded(conf)) ? 
             ZlibCompressor.class : BuiltInZlibDeflater.class;
   }
   
   /**
    * Return the appropriate implementation of the zlib compressor. 
    * 
+   * @param conf configuration
    * @return the appropriate implementation of the zlib compressor.
    */
-  public static Compressor getZlibCompressor() {
-    LOG.info("Creating a new ZlibCompressor");
-    return (nativeZlibLoaded) ? 
+  public static Compressor getZlibCompressor(Configuration conf) {
+    return (isNativeZlibLoaded(conf)) ? 
       new ZlibCompressor() : new BuiltInZlibDeflater(); 
   }
 
   /**
    * Return the appropriate type of the zlib decompressor. 
    * 
+   * @param conf configuration
    * @return the appropriate type of the zlib decompressor.
    */
-  public static Class getZlibDecompressorType() {
-    return (nativeZlibLoaded) ? 
+  public static Class getZlibDecompressorType(Configuration conf) {
+    return (isNativeZlibLoaded(conf)) ? 
             ZlibDecompressor.class : BuiltInZlibInflater.class;
   }
   
   /**
    * Return the appropriate implementation of the zlib decompressor. 
    * 
+   * @param conf configuration
    * @return the appropriate implementation of the zlib decompressor.
    */
-  public static Decompressor getZlibDecompressor() {
-    LOG.info("Creating a new ZlibDecompressor");
-    return (nativeZlibLoaded) ? 
+  public static Decompressor getZlibDecompressor(Configuration conf) {
+    return (isNativeZlibLoaded(conf)) ? 
       new ZlibDecompressor() : new BuiltInZlibInflater(); 
   }
   

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/util/NativeCodeLoader.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/util/NativeCodeLoader.java?view=diff&rev=555768&r1=555767&r2=555768
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/util/NativeCodeLoader.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/util/NativeCodeLoader.java Thu Jul 12 14:08:22
2007
@@ -19,6 +19,7 @@
 package org.apache.hadoop.util;
 
 import org.apache.commons.logging.*;
+import org.apache.hadoop.mapred.JobConf;
 
 /**
  * A helper to load the native hadoop code i.e. libhadoop.so.
@@ -60,6 +61,28 @@
    */
   public static boolean isNativeCodeLoaded() {
     return nativeCodeLoaded;
+  }
+
+  /**
+   * Return if native hadoop libraries, if present, can be used for this job.
+   * @param jobConf job configuration
+   * 
+   * @return <code>true</code> if native hadoop libraries, if present, can be

+   *         used for this job; <code>false</code> otherwise.
+   */
+  public boolean getLoadNativeLibraries(JobConf jobConf) {
+    return jobConf.getBoolean("hadoop.native.lib", true);
+  }
+  
+  /**
+   * Set if native hadoop libraries, if present, can be used for this job.
+   * 
+   * @param jobConf job configuration
+   * @param loadNativeLibraries can native hadoop libraries be loaded
+   */
+  public void setLoadNativeLibraries(JobConf jobConf, 
+                                     boolean loadNativeLibraries) {
+    jobConf.setBoolean("hadoop.native.lib", loadNativeLibraries);
   }
 
 }

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/io/TestSequenceFile.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/io/TestSequenceFile.java?view=diff&rev=555768&r1=555767&r2=555768
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/io/TestSequenceFile.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/io/TestSequenceFile.java Thu Jul 12 14:08:22
2007
@@ -49,7 +49,7 @@
   }
   
   public void testLzoSequenceFile() throws Exception {
-    if (LzoCodec.isNativeLzoLoaded()) {
+    if (LzoCodec.isNativeLzoLoaded(conf)) {
       LOG.info("Testing SequenceFile with LzoCodec");
       CompressionCodec lzoCodec = null;
       try {

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/io/compress/TestCodec.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/io/compress/TestCodec.java?view=diff&rev=555768&r1=555767&r2=555768
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/io/compress/TestCodec.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/io/compress/TestCodec.java Thu Jul 12 14:08:22
2007
@@ -40,28 +40,29 @@
   private static final Log LOG= 
     LogFactory.getLog("org.apache.hadoop.io.compress.TestCodec");
 
+  private Configuration conf = new Configuration();
   private int count = 10000;
   private int seed = new Random().nextInt();
   
   public void testDefaultCodec() throws IOException {
-    codecTest(seed, count, "org.apache.hadoop.io.compress.DefaultCodec");
+    codecTest(conf, seed, count, "org.apache.hadoop.io.compress.DefaultCodec");
   }
   
   public void testGzipCodec() throws IOException {
-    codecTest(seed, count, "org.apache.hadoop.io.compress.GzipCodec");
+    codecTest(conf, seed, count, "org.apache.hadoop.io.compress.GzipCodec");
   }
   
   public void testLzoCodec() throws IOException {
-    if (LzoCodec.isNativeLzoLoaded()) {
-      codecTest(seed, count, "org.apache.hadoop.io.compress.LzoCodec");
+    if (LzoCodec.isNativeLzoLoaded(conf)) {
+      codecTest(conf, seed, count, "org.apache.hadoop.io.compress.LzoCodec");
     }
   }
   
-  private static void codecTest(int seed, int count, String codecClass) 
+  private static void codecTest(Configuration conf, int seed, int count, 
+                                String codecClass) 
     throws IOException {
     
     // Create the codec
-    Configuration conf = new Configuration();
     CompressionCodec codec = null;
     try {
       codec = (CompressionCodec)
@@ -144,8 +145,9 @@
         }
       }
 
+      Configuration conf = new Configuration();
       int seed = 0;
-      codecTest(seed, count, codecClass);
+      codecTest(conf, seed, count, codecClass);
     } catch (Exception e) {
       System.err.println("Caught: " + e);
       e.printStackTrace();



Mime
View raw message