Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 9B9021885A for ; Sat, 19 Dec 2015 02:41:52 +0000 (UTC) Received: (qmail 83427 invoked by uid 500); 19 Dec 2015 02:41:51 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 83287 invoked by uid 500); 19 Dec 2015 02:41:51 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 83260 invoked by uid 99); 19 Dec 2015 02:41:51 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 19 Dec 2015 02:41:51 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 7EF16E03C7; Sat, 19 Dec 2015 02:41:51 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: cdouglas@apache.org To: common-commits@hadoop.apache.org Date: Sat, 19 Dec 2015 02:41:53 -0000 Message-Id: <032cc22eea6c46e0bef26c89b00ed695@git.apache.org> In-Reply-To: <51ae713922f6449fba4b20252a8e3571@git.apache.org> References: <51ae713922f6449fba4b20252a8e3571@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [3/3] hadoop git commit: HADOOP-12636. Prevent ServiceLoader failure init for unused FileSystems. Contributed by Inigo Goiri HADOOP-12636. Prevent ServiceLoader failure init for unused FileSystems. Contributed by Inigo Goiri (cherry picked from commit 8652cce5b21825f6f835e4ea31de82eb59fb06c1) (cherry picked from commit d984ba4fd73df64b7dca1ec10b35956e4ae9bd57) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4150d08a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4150d08a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4150d08a Branch: refs/heads/branch-2.8 Commit: 4150d08a4b44d2f61cd183fbeb4523721a5c2a77 Parents: d6f1d3b Author: Chris Douglas Authored: Fri Dec 18 18:21:52 2015 -0800 Committer: Chris Douglas Committed: Fri Dec 18 18:31:36 2015 -0800 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 5 +++-- .../java/org/apache/hadoop/fs/FileSystem.java | 18 ++++++++++++++++-- .../hadoop/fs/TestFileSystemInitialization.java | 16 ++++++++++++++++ 3 files changed, 35 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4150d08a/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index e22da88..247124f 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -765,8 +765,6 @@ Release 2.8.0 - UNRELEASED HADOOP-12656. MiniKdc throws "address in use" BindException. (Wei-Chiu Chuang via Arpit Agarwal) - OPTIMIZATIONS - HADOOP-12051. ProtobufRpcEngine.invoke() should use Exception.toString() over getMessage() in logging/span events. (Varun Saxena via stevel) @@ -882,6 +880,9 @@ Release 2.8.0 - UNRELEASED HADOOP-10729. Add tests for PB RPC in case version mismatch of client and server. (Junping Du via wheat9) + HADOOP-12636. Prevent ServiceLoader failure init for unused FileSystems. + (Inigo Goiri via cdouglas) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/4150d08a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index 8ed96bf..2102862 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -36,6 +36,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; +import java.util.ServiceConfigurationError; import java.util.ServiceLoader; import java.util.Set; import java.util.Stack; @@ -62,6 +63,7 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.ClassUtil; import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.ReflectionUtils; @@ -2708,8 +2710,20 @@ public abstract class FileSystem extends Configured implements Closeable { synchronized (FileSystem.class) { if (!FILE_SYSTEMS_LOADED) { ServiceLoader serviceLoader = ServiceLoader.load(FileSystem.class); - for (FileSystem fs : serviceLoader) { - SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass()); + Iterator it = serviceLoader.iterator(); + while (it.hasNext()) { + FileSystem fs = null; + try { + fs = it.next(); + try { + SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass()); + } catch (Exception e) { + LOG.warn("Cannot load: " + fs + " from " + + ClassUtil.findContainingJar(fs.getClass()), e); + } + } catch (ServiceConfigurationError ee) { + LOG.warn("Cannot load filesystem", ee); + } } FILE_SYSTEMS_LOADED = true; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4150d08a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java index d3fceec..18e8b01 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java @@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration; import java.io.IOException; import java.net.URL; +import java.util.ServiceConfigurationError; import org.junit.Test; import static org.junit.Assert.*; @@ -43,4 +44,19 @@ public class TestFileSystemInitialization { assertFalse(false); } } + + @Test + public void testMissingLibraries() { + boolean catched = false; + try { + Configuration conf = new Configuration(); + FileSystem.getFileSystemClass("s3a", conf); + } catch (Exception e) { + catched = true; + } catch (ServiceConfigurationError e) { + // S3A shouldn't find AWS SDK and fail + catched = true; + } + assertTrue(catched); + } }