Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 6F6E8200C6F for ; Tue, 9 May 2017 21:13:47 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 65D34160BB6; Tue, 9 May 2017 19:13:47 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 85E34160B9A for ; Tue, 9 May 2017 21:13:46 +0200 (CEST) Received: (qmail 55610 invoked by uid 500); 9 May 2017 19:13:45 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 55601 invoked by uid 99); 9 May 2017 19:13:45 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 09 May 2017 19:13:45 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8E0E4DFDAC; Tue, 9 May 2017 19:13:45 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: apurtell@apache.org To: commits@hbase.apache.org Message-Id: <39a62f9c339f4ffebb6420e418ca48fe@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hbase git commit: HBASE-18017 Reduce frequency of setStoragePolicy failure warnings Date: Tue, 9 May 2017 19:13:45 +0000 (UTC) archived-at: Tue, 09 May 2017 19:13:47 -0000 Repository: hbase Updated Branches: refs/heads/master 951b23a44 -> c38bf1244 HBASE-18017 Reduce frequency of setStoragePolicy failure warnings Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c38bf124 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c38bf124 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c38bf124 Branch: refs/heads/master Commit: c38bf12444aca77c7cb12637147c07dc711acbe9 Parents: 951b23a Author: Andrew Purtell Authored: Mon May 8 15:05:59 2017 -0700 Committer: Andrew Purtell Committed: Tue May 9 12:08:05 2017 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hbase/util/FSUtils.java | 67 +++++++++++++------- 1 file changed, 44 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/c38bf124/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java index c78ba06..284b786 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java @@ -152,6 +152,9 @@ public abstract class FSUtils { setStoragePolicy(fs, path, storagePolicy); } + private static final Map warningMap = + new ConcurrentHashMap(); + /** * Sets storage policy for given path. * If the passed path is a directory, we'll set the storage policy for all files @@ -187,17 +190,20 @@ public abstract class FSUtils { try { distributed = isDistributedFileSystem(fs); } catch (IOException ioe) { - // This should NEVER happen. - LOG.warn("Failed setStoragePolicy=" + trimmedStoragePolicy + " on path=" + - path + "; failed isDFS test", ioe); + if (!warningMap.containsKey(fs)) { + warningMap.put(fs, true); + LOG.warn("FileSystem isn't an instance of DistributedFileSystem; presuming it doesn't " + + "support setStoragePolicy. Unable to set storagePolicy=" + trimmedStoragePolicy + + " on path=" + path); + } else if (LOG.isDebugEnabled()) { + LOG.debug("FileSystem isn't an instance of DistributedFileSystem; presuming it doesn't " + + "support setStoragePolicy. Unable to set storagePolicy=" + trimmedStoragePolicy + + " on path=" + path); + } return; } if (distributed) { invokeSetStoragePolicy(fs, path, trimmedStoragePolicy); - } else { - LOG.info("FileSystem isn't an instance of DistributedFileSystem; presuming it doesn't " + - "support setStoragePolicy. Unable to set storagePolicy=" + trimmedStoragePolicy + - " on path=" + path); } } @@ -209,41 +215,56 @@ public abstract class FSUtils { Method m = null; try { m = fs.getClass().getDeclaredMethod("setStoragePolicy", - new Class[] { Path.class, String.class }); + new Class[] { Path.class, String.class }); m.setAccessible(true); } catch (NoSuchMethodException e) { - LOG.info("FileSystem doesn't support setStoragePolicy; HDFS-6584 not available " - + "(hadoop-2.6.0+): " + e.getMessage()); + final String msg = "FileSystem doesn't support setStoragePolicy; HDFS-6584 not available"; + if (!warningMap.containsKey(fs)) { + warningMap.put(fs, true); + LOG.warn(msg, e); + } else if (LOG.isDebugEnabled()) { + LOG.debug(msg, e); + } + m = null; } catch (SecurityException e) { - LOG.info("Don't have access to setStoragePolicy on FileSystems; HDFS-6584 not available " - + "(hadoop-2.6.0+): ", e); + final String msg = "No access to setStoragePolicy on FileSystem; HDFS-6584 not available"; + if (!warningMap.containsKey(fs)) { + warningMap.put(fs, true); + LOG.warn(msg, e); + } else if (LOG.isDebugEnabled()) { + LOG.debug(msg, e); + } m = null; // could happen on setAccessible() } if (m != null) { try { m.invoke(fs, path, storagePolicy); - LOG.info("Set storagePolicy=" + storagePolicy + " for path=" + path); + if (LOG.isDebugEnabled()) { + LOG.debug("Set storagePolicy=" + storagePolicy + " for path=" + path); + } } catch (Exception e) { + // This swallows FNFE, should we be throwing it? seems more likely to indicate dev + // misuse than a runtime problem with HDFS. + if (!warningMap.containsKey(fs)) { + warningMap.put(fs, true); + LOG.warn("Unable to set storagePolicy=" + storagePolicy + " for path=" + path, e); + } else if (LOG.isDebugEnabled()) { + LOG.debug("Unable to set storagePolicy=" + storagePolicy + " for path=" + path, e); + } // check for lack of HDFS-7228 - boolean probablyBadPolicy = false; if (e instanceof InvocationTargetException) { final Throwable exception = e.getCause(); if (exception instanceof RemoteException && HadoopIllegalArgumentException.class.getName().equals( - ((RemoteException)exception).getClassName())) { - LOG.warn("Given storage policy, '" + storagePolicy + "', was rejected and probably " + + ((RemoteException)exception).getClassName())) { + if (LOG.isDebugEnabled()) { + LOG.debug("Given storage policy, '" +storagePolicy +"', was rejected and probably " + "isn't a valid policy for the version of Hadoop you're running. I.e. if you're " + "trying to use SSD related policies then you're likely missing HDFS-7228. For " + "more information see the 'ArchivalStorage' docs for your Hadoop release."); - LOG.debug("More information about the invalid storage policy.", exception); - probablyBadPolicy = true; + } } } - if (!probablyBadPolicy) { - // This swallows FNFE, should we be throwing it? seems more likely to indicate dev - // misuse than a runtime problem with HDFS. - LOG.warn("Unable to set storagePolicy=" + storagePolicy + " for path=" + path, e); - } } } }