hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ehi...@apache.org
Subject [06/50] hadoop git commit: HADOOP-15612. Improve exception when tfile fails to load LzoCodec. (gera)
Date Tue, 31 Jul 2018 09:25:48 GMT
HADOOP-15612. Improve exception when tfile fails to load LzoCodec. (gera)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6bec03cf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6bec03cf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6bec03cf

Branch: refs/heads/HDFS-12090
Commit: 6bec03cfc8bdcf6aa3df9c22231ab959ba31f2f5
Parents: ea2c6c8
Author: Gera Shegalov <gera@apache.org>
Authored: Tue Jul 17 00:05:39 2018 -0700
Committer: Gera Shegalov <gera@apache.org>
Committed: Tue Jul 24 14:32:30 2018 -0700

----------------------------------------------------------------------
 .../hadoop/io/file/tfile/Compression.java       | 31 +++++++++++-------
 .../hadoop/io/file/tfile/TestCompression.java   | 34 +++++++++++++++++++-
 2 files changed, 53 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bec03cf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Compression.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Compression.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Compression.java
index fa85ed7..c4347e0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Compression.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Compression.java
@@ -5,9 +5,9 @@
  * licenses this file to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  * http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
@@ -24,6 +24,7 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.ArrayList;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.compress.CodecPool;
 import org.apache.hadoop.io.compress.CompressionCodec;
@@ -78,25 +79,33 @@ public final class Compression {
   public enum Algorithm {
     LZO(TFile.COMPRESSION_LZO) {
       private transient boolean checked = false;
+      private transient ClassNotFoundException cnf;
+      private transient boolean reinitCodecInTests;
       private static final String defaultClazz =
           "org.apache.hadoop.io.compress.LzoCodec";
+      private transient String clazz;
       private transient CompressionCodec codec = null;
 
+      private String getLzoCodecClass() {
+        String extClazzConf = conf.get(CONF_LZO_CLASS);
+        String extClazz = (extClazzConf != null) ?
+            extClazzConf : System.getProperty(CONF_LZO_CLASS);
+        return (extClazz != null) ? extClazz : defaultClazz;
+      }
+
       @Override
       public synchronized boolean isSupported() {
-        if (!checked) {
+        if (!checked || reinitCodecInTests) {
           checked = true;
-          String extClazzConf = conf.get(CONF_LZO_CLASS);
-          String extClazz = (extClazzConf != null) ?
-              extClazzConf : System.getProperty(CONF_LZO_CLASS);
-          String clazz = (extClazz != null) ? extClazz : defaultClazz;
+          reinitCodecInTests = conf.getBoolean("test.reload.lzo.codec", false);
+          clazz = getLzoCodecClass();
           try {
             LOG.info("Trying to load Lzo codec class: " + clazz);
             codec =
                 (CompressionCodec) ReflectionUtils.newInstance(Class
                     .forName(clazz), conf);
           } catch (ClassNotFoundException e) {
-            // that is okay
+            cnf = e;
           }
         }
         return codec != null;
@@ -105,9 +114,9 @@ public final class Compression {
       @Override
       CompressionCodec getCodec() throws IOException {
         if (!isSupported()) {
-          throw new IOException(
-              "LZO codec class not specified. Did you forget to set property "
-                  + CONF_LZO_CLASS + "?");
+          throw new IOException(String.format(
+              "LZO codec %s=%s could not be loaded", CONF_LZO_CLASS, clazz),
+                  cnf);
         }
 
         return codec;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bec03cf/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestCompression.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestCompression.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestCompression.java
index ff6c72a..b1bf077 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestCompression.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestCompression.java
@@ -17,14 +17,28 @@
  */
 package org.apache.hadoop.io.file.tfile;
 
-import org.junit.Test;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.*;
 
 import java.io.IOException;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 public class TestCompression {
 
+  @BeforeClass
+  public static void resetConfigBeforeAll() {
+    Compression.Algorithm.LZO.conf.setBoolean("test.reload.lzo.codec", true);
+  }
+
+  @AfterClass
+  public static void resetConfigAfterAll() {
+    Compression.Algorithm.LZO.conf.setBoolean("test.reload.lzo.codec", false);
+  }
+
   /**
    * Regression test for HADOOP-11418.
    * Verify we can set a LZO codec different from default LZO codec.
@@ -38,4 +52,22 @@ public class TestCompression {
     assertEquals(defaultCodec,
         Compression.Algorithm.LZO.getCodec().getClass().getName());
   }
+
+
+  @Test
+  public void testMisconfiguredLZOCodec() throws Exception {
+    // Dummy codec
+    String defaultCodec = "org.apache.hadoop.io.compress.InvalidLzoCodec";
+    Compression.Algorithm.conf.set(
+        Compression.Algorithm.CONF_LZO_CLASS, defaultCodec);
+    IOException ioEx = LambdaTestUtils.intercept(
+        IOException.class,
+        defaultCodec,
+        () -> Compression.Algorithm.LZO.getCodec());
+
+    if (!(ioEx.getCause() instanceof ClassNotFoundException)) {
+      throw ioEx;
+    }
+  }
+
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message