hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject hbase git commit: HBASE-18017 Reduce frequency of setStoragePolicy failure warnings
Date Tue, 09 May 2017 19:13:45 GMT
Repository: hbase
Updated Branches:
  refs/heads/master 951b23a44 -> c38bf1244


HBASE-18017 Reduce frequency of setStoragePolicy failure warnings


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c38bf124
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c38bf124
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c38bf124

Branch: refs/heads/master
Commit: c38bf12444aca77c7cb12637147c07dc711acbe9
Parents: 951b23a
Author: Andrew Purtell <apurtell@apache.org>
Authored: Mon May 8 15:05:59 2017 -0700
Committer: Andrew Purtell <apurtell@apache.org>
Committed: Tue May 9 12:08:05 2017 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hbase/util/FSUtils.java   | 67 +++++++++++++-------
 1 file changed, 44 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/c38bf124/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index c78ba06..284b786 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -152,6 +152,9 @@ public abstract class FSUtils {
     setStoragePolicy(fs, path, storagePolicy);
   }
 
+  private static final Map<FileSystem, Boolean> warningMap =
+      new ConcurrentHashMap<FileSystem, Boolean>();
+
   /**
    * Sets storage policy for given path.
    * If the passed path is a directory, we'll set the storage policy for all files
@@ -187,17 +190,20 @@ public abstract class FSUtils {
     try {
       distributed = isDistributedFileSystem(fs);
     } catch (IOException ioe) {
-      // This should NEVER happen.
-      LOG.warn("Failed setStoragePolicy=" + trimmedStoragePolicy + " on path=" +
-          path + "; failed isDFS test", ioe);
+      if (!warningMap.containsKey(fs)) {
+        warningMap.put(fs, true);
+        LOG.warn("FileSystem isn't an instance of DistributedFileSystem; presuming it doesn't
"
+            + "support setStoragePolicy. Unable to set storagePolicy=" + trimmedStoragePolicy
+            + " on path=" + path);
+      } else if (LOG.isDebugEnabled()) {
+        LOG.debug("FileSystem isn't an instance of DistributedFileSystem; presuming it doesn't
"
+            + "support setStoragePolicy. Unable to set storagePolicy=" + trimmedStoragePolicy
+            + " on path=" + path);
+      }
       return;
     }
     if (distributed) {
       invokeSetStoragePolicy(fs, path, trimmedStoragePolicy);
-    } else {
-      LOG.info("FileSystem isn't an instance of DistributedFileSystem; presuming it doesn't
" +
-          "support setStoragePolicy. Unable to set storagePolicy=" + trimmedStoragePolicy
+
-          " on path=" + path);
     }
   }
 
@@ -209,41 +215,56 @@ public abstract class FSUtils {
     Method m = null;
     try {
       m = fs.getClass().getDeclaredMethod("setStoragePolicy",
-          new Class<?>[] { Path.class, String.class });
+        new Class<?>[] { Path.class, String.class });
       m.setAccessible(true);
     } catch (NoSuchMethodException e) {
-      LOG.info("FileSystem doesn't support setStoragePolicy; HDFS-6584 not available "
-          + "(hadoop-2.6.0+): " + e.getMessage());
+      final String msg = "FileSystem doesn't support setStoragePolicy; HDFS-6584 not available";
+      if (!warningMap.containsKey(fs)) {
+        warningMap.put(fs, true);
+        LOG.warn(msg, e);
+      } else if (LOG.isDebugEnabled()) {
+        LOG.debug(msg, e);
+      }
+      m = null;
     } catch (SecurityException e) {
-      LOG.info("Don't have access to setStoragePolicy on FileSystems; HDFS-6584 not available
"
-          + "(hadoop-2.6.0+): ", e);
+      final String msg = "No access to setStoragePolicy on FileSystem; HDFS-6584 not available";
+      if (!warningMap.containsKey(fs)) {
+        warningMap.put(fs, true);
+        LOG.warn(msg, e);
+      } else if (LOG.isDebugEnabled()) {
+        LOG.debug(msg, e);
+      }
       m = null; // could happen on setAccessible()
     }
     if (m != null) {
       try {
         m.invoke(fs, path, storagePolicy);
-        LOG.info("Set storagePolicy=" + storagePolicy + " for path=" + path);
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Set storagePolicy=" + storagePolicy + " for path=" + path);
+        }
       } catch (Exception e) {
+        // This swallows FNFE, should we be throwing it? seems more likely to indicate dev
+        // misuse than a runtime problem with HDFS.
+        if (!warningMap.containsKey(fs)) {
+          warningMap.put(fs, true);
+          LOG.warn("Unable to set storagePolicy=" + storagePolicy + " for path=" + path,
e);
+        } else if (LOG.isDebugEnabled()) {
+          LOG.debug("Unable to set storagePolicy=" + storagePolicy + " for path=" + path,
e);
+        }
         // check for lack of HDFS-7228
-        boolean probablyBadPolicy = false;
         if (e instanceof InvocationTargetException) {
           final Throwable exception = e.getCause();
           if (exception instanceof RemoteException &&
               HadoopIllegalArgumentException.class.getName().equals(
-                  ((RemoteException)exception).getClassName())) {
-            LOG.warn("Given storage policy, '" + storagePolicy + "', was rejected and probably
" +
+                ((RemoteException)exception).getClassName())) {
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Given storage policy, '" +storagePolicy +"', was rejected and probably
" +
                 "isn't a valid policy for the version of Hadoop you're running. I.e. if you're
" +
                 "trying to use SSD related policies then you're likely missing HDFS-7228.
For " +
                 "more information see the 'ArchivalStorage' docs for your Hadoop release.");
-            LOG.debug("More information about the invalid storage policy.", exception);
-            probablyBadPolicy = true;
+            }
           }
         }
-        if (!probablyBadPolicy) {
-          // This swallows FNFE, should we be throwing it? seems more likely to indicate
dev
-          // misuse than a runtime problem with HDFS.
-          LOG.warn("Unable to set storagePolicy=" + storagePolicy + " for path=" + path,
e);
-        }
       }
     }
   }


Mime
View raw message