hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-12848 Utilize Flash storage for WAL
Date Thu, 15 Jan 2015 16:52:35 GMT
Repository: hbase
Updated Branches:
  refs/heads/master 71184309e -> eafc07a06


HBASE-12848 Utilize Flash storage for WAL


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/eafc07a0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/eafc07a0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/eafc07a0

Branch: refs/heads/master
Commit: eafc07a06d03c00e17bd476fa2b84ba7c8924b1e
Parents: 7118430
Author: tedyu <yuzhihong@gmail.com>
Authored: Thu Jan 15 08:52:30 2015 -0800
Committer: tedyu <yuzhihong@gmail.com>
Committed: Thu Jan 15 08:52:30 2015 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/hbase/HConstants.java     | 10 +++++
 .../hadoop/hbase/regionserver/wal/FSHLog.java   |  2 +
 .../org/apache/hadoop/hbase/util/FSUtils.java   | 39 ++++++++++++++++++++
 3 files changed, 51 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/eafc07a0/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index eee5e83..8585299 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -918,6 +918,16 @@ public final class HConstants {
   public static final String ENABLE_WAL_COMPRESSION =
     "hbase.regionserver.wal.enablecompression";
 
+  /** Configuration name of WAL storage policy
+   * Valid values are:
+   *  NONE: no preference in destination of replicas
+   *  ONE_SSD: place only one replica in SSD and the remaining in default storage
+   *  and ALL_SSD: place all replica on SSD
+   *  
+   * See http://hadoop.apache.org/docs/r2.6.0/hadoop-project-dist/hadoop-hdfs/ArchivalStorage.html*/
+  public static final String WAL_STORAGE_POLICY = "hbase.wal.storage.policy";
+  public static final String DEFAULT_WAL_STORAGE_POLICY = "NONE";
+
   /** Region in Transition metrics threshold time */
   public static final String METRICS_RIT_STUCK_WARNING_THRESHOLD="hbase.metrics.rit.stuck.warning.threshold";
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/eafc07a0/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
index 1fad93d..3ba9b4c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
@@ -491,6 +491,8 @@ public class FSHLog implements WAL {
       throw new IllegalArgumentException("wal suffix must start with '" + WAL_FILE_NAME_DELIMITER
+
           "' but instead was '" + suffix + "'");
     }
+    FSUtils.setStoragePolicy(fs, conf, this.fullPathLogDir, HConstants.WAL_STORAGE_POLICY,
+      HConstants.DEFAULT_WAL_STORAGE_POLICY);
     this.logFileSuffix = (suffix == null) ? "" : URLEncoder.encode(suffix, "UTF8");
     this.prefixPathStr = new Path(fullPathLogDir,
         logFilePrefix + WAL_FILE_NAME_DELIMITER).toString();

http://git-wip-us.apache.org/repos/asf/hbase/blob/eafc07a0/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 7cda55d..f5b0269 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -102,6 +102,45 @@ public abstract class FSUtils {
     super();
   }
 
+  /*
+   * Sets storage policy for given path according to config setting
+   * @param fs
+   * @param conf
+   * @param path the Path whose storage policy is to be set
+   * @param policyKey
+   * @param defaultPolicy
+   */
+  public static void setStoragePolicy(final FileSystem fs, final Configuration conf,
+      final Path path, final String policyKey, final String defaultPolicy) {
+    String storagePolicy = conf.get(policyKey, defaultPolicy).toUpperCase();
+    if (!storagePolicy.equals(defaultPolicy) &&
+        fs instanceof DistributedFileSystem) {
+      DistributedFileSystem dfs = (DistributedFileSystem)fs;
+      Class<? extends DistributedFileSystem> dfsClass = dfs.getClass();
+      Method m = null;
+      try {
+        m = dfsClass.getDeclaredMethod("setStoragePolicy",
+            new Class<?>[] { Path.class, String.class });
+        m.setAccessible(true);
+      } catch (NoSuchMethodException e) {
+        LOG.info("FileSystem doesn't support"
+            + " setStoragePolicy; --HDFS-7228 not available");
+      } catch (SecurityException e) {
+        LOG.info("Doesn't have access to setStoragePolicy on "
+            + "FileSystems --HDFS-7228 not available", e);
+        m = null; // could happen on setAccessible()
+      }
+      if (m != null) {
+        try {
+          m.invoke(dfs, path, storagePolicy);
+          LOG.info("set " + storagePolicy + " for " + path);
+        } catch (Exception e) {
+          LOG.warn("Unable to set " + storagePolicy + " for " + path, e);
+        }
+      }
+    }
+  }
+
   /**
    * Compare of path component. Does not consider schema; i.e. if schemas different but <code>path
    * <code> starts with <code>rootPath<code>, then the function returns
true


Mime
View raw message