hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject [3/3] hadoop git commit: HADOOP-12636. Prevent ServiceLoader failure init for unused FileSystems. Contributed by Inigo Goiri
Date Sat, 19 Dec 2015 02:41:53 GMT
HADOOP-12636. Prevent ServiceLoader failure init for unused FileSystems. Contributed by Inigo
Goiri

(cherry picked from commit 8652cce5b21825f6f835e4ea31de82eb59fb06c1)
(cherry picked from commit d984ba4fd73df64b7dca1ec10b35956e4ae9bd57)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4150d08a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4150d08a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4150d08a

Branch: refs/heads/branch-2.8
Commit: 4150d08a4b44d2f61cd183fbeb4523721a5c2a77
Parents: d6f1d3b
Author: Chris Douglas <cdouglas@apache.org>
Authored: Fri Dec 18 18:21:52 2015 -0800
Committer: Chris Douglas <cdouglas@apache.org>
Committed: Fri Dec 18 18:31:36 2015 -0800

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt   |  5 +++--
 .../java/org/apache/hadoop/fs/FileSystem.java     | 18 ++++++++++++++++--
 .../hadoop/fs/TestFileSystemInitialization.java   | 16 ++++++++++++++++
 3 files changed, 35 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4150d08a/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index e22da88..247124f 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -765,8 +765,6 @@ Release 2.8.0 - UNRELEASED
     HADOOP-12656. MiniKdc throws "address in use" BindException.
     (Wei-Chiu Chuang via Arpit Agarwal)
 
-  OPTIMIZATIONS
-
     HADOOP-12051. ProtobufRpcEngine.invoke() should use Exception.toString()
     over getMessage() in logging/span events. (Varun Saxena via stevel)
 
@@ -882,6 +880,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-10729. Add tests for PB RPC in case version mismatch of client and
     server. (Junping Du via wheat9)
 
+    HADOOP-12636. Prevent ServiceLoader failure init for unused FileSystems.
+    (Inigo Goiri via cdouglas)
+
 Release 2.7.3 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4150d08a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 8ed96bf..2102862 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -36,6 +36,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
+import java.util.ServiceConfigurationError;
 import java.util.ServiceLoader;
 import java.util.Set;
 import java.util.Stack;
@@ -62,6 +63,7 @@ import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.ClassUtil;
 import org.apache.hadoop.util.DataChecksum;
 import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.util.ReflectionUtils;
@@ -2708,8 +2710,20 @@ public abstract class FileSystem extends Configured implements Closeable
{
     synchronized (FileSystem.class) {
       if (!FILE_SYSTEMS_LOADED) {
         ServiceLoader<FileSystem> serviceLoader = ServiceLoader.load(FileSystem.class);
-        for (FileSystem fs : serviceLoader) {
-          SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass());
+        Iterator<FileSystem> it = serviceLoader.iterator();
+        while (it.hasNext()) {
+          FileSystem fs = null;
+          try {
+            fs = it.next();
+            try {
+              SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass());
+            } catch (Exception e) {
+              LOG.warn("Cannot load: " + fs + " from " +
+                  ClassUtil.findContainingJar(fs.getClass()), e);
+            }
+          } catch (ServiceConfigurationError ee) {
+            LOG.warn("Cannot load filesystem", ee);
+          }
         }
         FILE_SYSTEMS_LOADED = true;
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4150d08a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
index d3fceec..18e8b01 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration;
 
 import java.io.IOException;
 import java.net.URL;
+import java.util.ServiceConfigurationError;
 
 import org.junit.Test;
 import static org.junit.Assert.*;
@@ -43,4 +44,19 @@ public class TestFileSystemInitialization {
       assertFalse(false);
     }
   }
+
+  @Test
+  public void testMissingLibraries() {
+    boolean catched = false;
+    try {
+      Configuration conf = new Configuration();
+      FileSystem.getFileSystemClass("s3a", conf);
+    } catch (Exception e) {
+      catched = true;
+    } catch (ServiceConfigurationError e) {
+      // S3A shouldn't find AWS SDK and fail
+      catched = true;
+    }
+    assertTrue(catched);
+  }
 }


Mime
View raw message