hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject [1/2] hadoop git commit: HADOOP-12636. Prevent ServiceLoader failure init for unused FileSystems. Contributed by Inigo Goiri. Cherry picked from commit #8652cce5b2
Date Mon, 11 Jul 2016 22:32:03 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7.3 01aa7d8f2 -> 490bcda71


HADOOP-12636. Prevent ServiceLoader failure init for unused FileSystems. Contributed by Inigo
Goiri.
Cherry picked from commit #8652cce5b2

(cherry picked from commit 7595d89612f4916ab29a6ac4e07374771fcbc94f)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fcd058ed
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fcd058ed
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fcd058ed

Branch: refs/heads/branch-2.7.3
Commit: fcd058edb60c7a02ad07d0fb8ccf85befe212308
Parents: 01aa7d8
Author: Chris Douglas <cdouglas@apache.org>
Authored: Fri Dec 18 18:21:52 2015 -0800
Committer: Vinod Kumar Vavilapalli <vinodkv@apache.org>
Committed: Mon Jul 11 15:28:01 2016 -0700

----------------------------------------------------------------------
 .../java/org/apache/hadoop/fs/FileSystem.java     | 18 ++++++++++++++++--
 .../hadoop/fs/TestFileSystemInitialization.java   | 16 ++++++++++++++++
 2 files changed, 32 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcd058ed/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index fac3c40..2e684f5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -35,6 +35,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
+import java.util.ServiceConfigurationError;
 import java.util.ServiceLoader;
 import java.util.Set;
 import java.util.Stack;
@@ -61,6 +62,7 @@ import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.ClassUtil;
 import org.apache.hadoop.util.DataChecksum;
 import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.util.ReflectionUtils;
@@ -2622,8 +2624,20 @@ public abstract class FileSystem extends Configured implements Closeable
{
     synchronized (FileSystem.class) {
       if (!FILE_SYSTEMS_LOADED) {
         ServiceLoader<FileSystem> serviceLoader = ServiceLoader.load(FileSystem.class);
-        for (FileSystem fs : serviceLoader) {
-          SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass());
+        Iterator<FileSystem> it = serviceLoader.iterator();
+        while (it.hasNext()) {
+          FileSystem fs = null;
+          try {
+            fs = it.next();
+            try {
+              SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass());
+            } catch (Exception e) {
+              LOG.warn("Cannot load: " + fs + " from " +
+                  ClassUtil.findContainingJar(fs.getClass()), e);
+            }
+          } catch (ServiceConfigurationError ee) {
+            LOG.warn("Cannot load filesystem", ee);
+          }
         }
         FILE_SYSTEMS_LOADED = true;
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcd058ed/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
index d3fceec..18e8b01 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration;
 
 import java.io.IOException;
 import java.net.URL;
+import java.util.ServiceConfigurationError;
 
 import org.junit.Test;
 import static org.junit.Assert.*;
@@ -43,4 +44,19 @@ public class TestFileSystemInitialization {
       assertFalse(false);
     }
   }
+
+  @Test
+  public void testMissingLibraries() {
+    boolean catched = false;
+    try {
+      Configuration conf = new Configuration();
+      FileSystem.getFileSystemClass("s3a", conf);
+    } catch (Exception e) {
+      catched = true;
+    } catch (ServiceConfigurationError e) {
+      // S3A shouldn't find AWS SDK and fail
+      catched = true;
+    }
+    assertTrue(catched);
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message