hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From md...@apache.org
Subject hbase git commit: HBASE-19289 Add flag to disable stream capability enforcement
Date Thu, 14 Dec 2017 18:20:13 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-2 eac13e231 -> 2952cc7de


HBASE-19289 Add flag to disable stream capability enforcement

Signed-off-by: Josh Elser <elserj@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2952cc7d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2952cc7d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2952cc7d

Branch: refs/heads/branch-2
Commit: 2952cc7dea4101d85eaf948c44d5e7f5b864f376
Parents: eac13e2
Author: Mike Drob <mdrob@apache.org>
Authored: Tue Dec 5 14:25:37 2017 -0600
Committer: Mike Drob <mdrob@apache.org>
Committed: Thu Dec 14 12:19:59 2017 -0600

----------------------------------------------------------------------
 .../java/org/apache/hadoop/hbase/util/CommonFSUtils.java    | 3 +++
 .../hbase/procedure2/store/wal/WALProcedureStore.java       | 6 ++++--
 hbase-procedure/src/test/resources/hbase-site.xml           | 9 +++++++++
 .../apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java | 5 +++--
 .../hadoop/hbase/regionserver/wal/ProtobufLogWriter.java    | 3 ++-
 hbase-server/src/test/resources/hbase-site.xml              | 9 +++++++++
 6 files changed, 30 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/2952cc7d/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
index 38ae476..2a83a5d 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
@@ -62,6 +62,9 @@ public abstract class CommonFSUtils {
   /** Parameter name for HBase WAL directory */
   public static final String HBASE_WAL_DIR = "hbase.wal.dir";
 
+  /** Parameter to disable stream capability enforcement checks */
+  public static final String UNSAFE_STREAM_CAPABILITY_ENFORCE = "hbase.unsafe.stream.capability.enforce";
+
   /** Full access permissions (starting point for a umask) */
   public static final String FULL_RWX_PERMISSIONS = "777";
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/2952cc7d/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
index f49833c..84cda65 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
@@ -131,6 +131,7 @@ public class WALProcedureStore extends ProcedureStoreBase {
   private final FileSystem fs;
   private final Path walDir;
   private final Path walArchiveDir;
+  private final boolean enforceStreamCapability;
 
   private final AtomicReference<Throwable> syncException = new AtomicReference<>();
   private final AtomicBoolean loading = new AtomicBoolean(true);
@@ -205,6 +206,7 @@ public class WALProcedureStore extends ProcedureStoreBase {
     this.walDir = walDir;
     this.walArchiveDir = walArchiveDir;
     this.fs = walDir.getFileSystem(conf);
+    this.enforceStreamCapability = conf.getBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE,
true);
 
     // Create the log directory for the procedure store
     if (!fs.exists(walDir)) {
@@ -1028,8 +1030,8 @@ public class WALProcedureStore extends ProcedureStoreBase {
     // ensure that we can provide the level of data safety we're configured
     // to provide.
     final String durability = useHsync ? "hsync" : "hflush";
-    if (!(CommonFSUtils.hasCapability(newStream, durability))) {
-      throw new IllegalStateException("The procedure WAL relies on the ability to " + durability
+
+    if (enforceStreamCapability && !(CommonFSUtils.hasCapability(newStream, durability)))
{
+        throw new IllegalStateException("The procedure WAL relies on the ability to " + durability
+
           " for proper operation during component failures, but the underlying filesystem
does " +
           "not support doing so. Please check the config value of '" + USE_HSYNC_CONF_KEY
+
           "' to set the desired level of robustness and ensure the config value of '" +

http://git-wip-us.apache.org/repos/asf/hbase/blob/2952cc7d/hbase-procedure/src/test/resources/hbase-site.xml
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/resources/hbase-site.xml b/hbase-procedure/src/test/resources/hbase-site.xml
index d350127..114ee8a 100644
--- a/hbase-procedure/src/test/resources/hbase-site.xml
+++ b/hbase-procedure/src/test/resources/hbase-site.xml
@@ -32,4 +32,13 @@
     procedure to have an owner
     </description>
   </property>
+  <property>
+    <name>hbase.unsafe.stream.capability.enforce</name>
+    <value>false</value>
+    <description>
+      Controls whether HBase will check for stream capabilities (hflush/hsync).
+      Disable this if you intend to run on LocalFileSystem.
+      WARNING: Doing so may expose you to additional risk of data loss!
+    </description>
+  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/hbase/blob/2952cc7d/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java
index 6a7e4fa..583759b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java
@@ -73,8 +73,9 @@ public final class AsyncFSOutputHelper {
     // After we create the stream but before we attempt to use it at all
     // ensure that we can provide the level of data safety we're configured
     // to provide.
-    if (!(CommonFSUtils.hasCapability(fsOut, "hflush") &&
-      CommonFSUtils.hasCapability(fsOut, "hsync"))) {
+    if (fs.getConf().getBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, true) &&
+        !(CommonFSUtils.hasCapability(fsOut, "hflush") &&
+          CommonFSUtils.hasCapability(fsOut, "hsync"))) {
       throw new CommonFSUtils.StreamLacksCapabilityException("hflush and hsync");
     }
     final ExecutorService flushExecutor =

http://git-wip-us.apache.org/repos/asf/hbase/blob/2952cc7d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java
index 64acfba..7a135c9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java
@@ -93,7 +93,8 @@ public class ProtobufLogWriter extends AbstractProtobufLogWriter
     this.output = fs.createNonRecursive(path, overwritable, bufferSize, replication, blockSize,
       null);
     // TODO Be sure to add a check for hsync if this branch includes HBASE-19024
-    if (!(CommonFSUtils.hasCapability(output, "hflush"))) {
+    if (fs.getConf().getBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, true) &&
+        !(CommonFSUtils.hasCapability(output, "hflush"))) {
       throw new StreamLacksCapabilityException("hflush");
     }
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/2952cc7d/hbase-server/src/test/resources/hbase-site.xml
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/resources/hbase-site.xml b/hbase-server/src/test/resources/hbase-site.xml
index 64a1964..dbdf776 100644
--- a/hbase-server/src/test/resources/hbase-site.xml
+++ b/hbase-server/src/test/resources/hbase-site.xml
@@ -158,4 +158,13 @@
     <name>hbase.hconnection.threads.keepalivetime</name>
     <value>3</value>
   </property>
+  <property>
+    <name>hbase.unsafe.stream.capability.enforce</name>
+    <value>false</value>
+    <description>
+      Controls whether HBase will check for stream capabilities (hflush/hsync).
+      Disable this if you intend to run on LocalFileSystem.
+      WARNING: Doing so may expose you to additional risk of data loss!
+    </description>
+  </property>
 </configuration>


Mime
View raw message