tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject git commit: TEZ-1474. detect missing native libraries for compression at the beginning of a task rather than at the end. Contributed by Prakash Ramachandran.
Date Fri, 22 Aug 2014 21:29:16 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.5 57bd9f7e3 -> f32db0ea9


TEZ-1474. detect missing native libraries for compression at the
beginning of a task rather than at the end. Contributed by Prakash
Ramachandran.


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/f32db0ea
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/f32db0ea
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/f32db0ea

Branch: refs/heads/branch-0.5
Commit: f32db0ea96369d602bf0c3ed7382b39f079da704
Parents: 57bd9f7
Author: Siddharth Seth <sseth@apache.org>
Authored: Fri Aug 22 14:28:18 2014 -0700
Committer: Siddharth Seth <sseth@apache.org>
Committed: Fri Aug 22 14:29:03 2014 -0700

----------------------------------------------------------------------
 .../library/common/sort/impl/ExternalSorter.java  | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/f32db0ea/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java
index 00f985e..bb8c912 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RawLocalFileSystem;
 import org.apache.hadoop.io.RawComparator;
 import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.Compressor;
 import org.apache.hadoop.io.compress.DefaultCodec;
 import org.apache.hadoop.io.serializer.SerializationFactory;
 import org.apache.hadoop.io.serializer.Serializer;
@@ -182,6 +183,23 @@ public abstract class ExternalSorter {
       Class<? extends CompressionCodec> codecClass =
           ConfigUtils.getIntermediateOutputCompressorClass(this.conf, DefaultCodec.class);
       codec = ReflectionUtils.newInstance(codecClass, this.conf);
+
+      if (codec != null) {
+        Class<? extends Compressor> compressorType = null;
+        Throwable cause = null;
+        try {
+          compressorType = codec.getCompressorType();
+        } catch (RuntimeException e) {
+          cause = e;
+        }
+        if (compressorType == null) {
+          String errMsg =
+              String.format("Unable to get CompressorType for codec (%s). This is most" +
+                      " likely due to missing native libraries for the codec.",
+                  conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC));
+          throw new IOException(errMsg, cause);
+        }
+      }
     } else {
       codec = null;
     }


Mime
View raw message