hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mberto...@apache.org
Subject git commit: HBASE-10935 support snapshot policy where flush memstore can be skipped to prevent production cluster freeze (Tianying Chang)
Date Mon, 02 Jun 2014 07:40:52 GMT
Repository: hbase
Updated Branches:
  refs/heads/0.98 7257dbc58 -> 60e705b84


HBASE-10935 support snapshot policy where flush memstore can be skipped to prevent production
cluster freeze (Tianying Chang)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/60e705b8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/60e705b8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/60e705b8

Branch: refs/heads/0.98
Commit: 60e705b8434b00fbe07b4373fcc21e3ce6881b11
Parents: 7257dbc
Author: Matteo Bertozzi <matteo.bertozzi@cloudera.com>
Authored: Mon Jun 2 08:14:42 2014 +0100
Committer: Matteo Bertozzi <matteo.bertozzi@cloudera.com>
Committed: Mon Jun 2 08:14:42 2014 +0100

----------------------------------------------------------------------
 .../apache/hadoop/hbase/client/HBaseAdmin.java  | 20 ++++++++
 .../hbase/protobuf/generated/HBaseProtos.java   | 44 ++++++++++-------
 hbase-protocol/src/main/protobuf/HBase.proto    |  1 +
 .../snapshot/FlushSnapshotSubprocedure.java     | 26 ++++++++--
 .../snapshot/RegionServerSnapshotManager.java   | 13 +++++
 .../snapshot/TestFlushSnapshotFromClient.java   | 52 ++++++++++++++++++++
 hbase-shell/src/main/ruby/hbase.rb              |  1 +
 hbase-shell/src/main/ruby/hbase/admin.rb        | 15 +++++-
 .../src/main/ruby/shell/commands/snapshot.rb    |  6 +--
 9 files changed, 153 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/60e705b8/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index df5d295..43b31d1 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -2598,6 +2598,26 @@ public class HBaseAdmin implements Abortable, Closeable {
   }
 
   /**
+   * Create snapshot for the given table of given flush type.
+   * <p>
+   * Snapshots are considered unique based on <b>the name of the snapshot</b>.
Attempts to take a
+   * snapshot with the same name (even a different type or with different parameters) will
fail with
+   * a {@link SnapshotCreationException} indicating the duplicate naming.
+   * <p>
+   * Snapshot names follow the same naming constraints as tables in HBase.
+   * @param snapshotName name of the snapshot to be created
+   * @param tableName name of the table for which snapshot is created
+   * @param flushType if the snapshot should be taken without flush memstore first
+   * @throws IOException if a remote or network exception occurs
+   * @throws SnapshotCreationException if snapshot creation failed
+   * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
+   */
+   public void snapshot(final byte[] snapshotName, final byte[] tableName,
+                       final SnapshotDescription.Type flushType) throws
+      IOException, SnapshotCreationException, IllegalArgumentException {
+      snapshot(Bytes.toString(snapshotName), Bytes.toString(tableName), flushType);
+  }
+  /**
    public void snapshot(final String snapshotName,
     * Create a timestamp consistent snapshot for the given table.
                         final byte[] tableName) throws IOException,

http://git-wip-us.apache.org/repos/asf/hbase/blob/60e705b8/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java
----------------------------------------------------------------------
diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java
index 238db31..9c0447e 100644
--- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java
+++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java
@@ -10476,6 +10476,10 @@ public final class HBaseProtos {
        * <code>FLUSH = 1;</code>
        */
       FLUSH(1, 1),
+      /**
+       * <code>SKIPFLUSH = 2;</code>
+       */
+      SKIPFLUSH(2, 2),
       ;
 
       /**
@@ -10486,6 +10490,10 @@ public final class HBaseProtos {
        * <code>FLUSH = 1;</code>
        */
       public static final int FLUSH_VALUE = 1;
+      /**
+       * <code>SKIPFLUSH = 2;</code>
+       */
+      public static final int SKIPFLUSH_VALUE = 2;
 
 
       public final int getNumber() { return value; }
@@ -10494,6 +10502,7 @@ public final class HBaseProtos {
         switch (value) {
           case 0: return DISABLED;
           case 1: return FLUSH;
+          case 2: return SKIPFLUSH;
           default: return null;
         }
       }
@@ -16241,26 +16250,27 @@ public final class HBaseProtos {
       ",\n\rNameBytesPair\022\014\n\004name\030\001 \002(\t\022\r\n\005value\030" +
       "\002 \001(\014\"/\n\016BytesBytesPair\022\r\n\005first\030\001 \002(\014\022\016"
+
       "\n\006second\030\002 \002(\014\",\n\rNameInt64Pair\022\014\n\004name\030" +
-      "\001 \001(\t\022\r\n\005value\030\002 \001(\003\"\256\001\n\023SnapshotDescrip" +
+      "\001 \001(\t\022\r\n\005value\030\002 \001(\003\"\275\001\n\023SnapshotDescrip" +
       "tion\022\014\n\004name\030\001 \002(\t\022\r\n\005table\030\002 \001(\t\022\030\n\rcre"
+
       "ation_time\030\003 \001(\003:\0010\022.\n\004type\030\004 \001(\0162\031.Snap" +
       "shotDescription.Type:\005FLUSH\022\017\n\007version\030\005" +
-      " \001(\005\"\037\n\004Type\022\014\n\010DISABLED\020\000\022\t\n\005FLUSH\020\001\"}\n",
-      "\024ProcedureDescription\022\021\n\tsignature\030\001 \002(\t" +
-      "\022\020\n\010instance\030\002 \001(\t\022\030\n\rcreation_time\030\003 \001(" +
-      "\003:\0010\022&\n\rconfiguration\030\004 \003(\0132\017.NameString" +
-      "Pair\"\n\n\010EmptyMsg\"\033\n\007LongMsg\022\020\n\010long_msg\030" +
-      "\001 \002(\003\"\037\n\tDoubleMsg\022\022\n\ndouble_msg\030\001 \002(\001\"\'" +
-      "\n\rBigDecimalMsg\022\026\n\016bigdecimal_msg\030\001 \002(\014\"" +
-      "5\n\004UUID\022\026\n\016least_sig_bits\030\001 \002(\004\022\025\n\rmost_" +
-      "sig_bits\030\002 \002(\004\"K\n\023NamespaceDescriptor\022\014\n" +
-      "\004name\030\001 \002(\014\022&\n\rconfiguration\030\002 \003(\0132\017.Nam" +
-      "eStringPair\"$\n\020RegionServerInfo\022\020\n\010infoP",
-      "ort\030\001 \001(\005*r\n\013CompareType\022\010\n\004LESS\020\000\022\021\n\rLE"
+
-      "SS_OR_EQUAL\020\001\022\t\n\005EQUAL\020\002\022\r\n\tNOT_EQUAL\020\003\022" +
-      "\024\n\020GREATER_OR_EQUAL\020\004\022\013\n\007GREATER\020\005\022\t\n\005NO" +
-      "_OP\020\006B>\n*org.apache.hadoop.hbase.protobu" +
-      "f.generatedB\013HBaseProtosH\001\240\001\001"
+      " \001(\005\".\n\004Type\022\014\n\010DISABLED\020\000\022\t\n\005FLUSH\020\001\022\r\n",
+      "\tSKIPFLUSH\020\002\"}\n\024ProcedureDescription\022\021\n\t" +
+      "signature\030\001 \002(\t\022\020\n\010instance\030\002 \001(\t\022\030\n\rcre" +
+      "ation_time\030\003 \001(\003:\0010\022&\n\rconfiguration\030\004 \003" +
+      "(\0132\017.NameStringPair\"\n\n\010EmptyMsg\"\033\n\007LongM" +
+      "sg\022\020\n\010long_msg\030\001 \002(\003\"\037\n\tDoubleMsg\022\022\n\ndou" +
+      "ble_msg\030\001 \002(\001\"\'\n\rBigDecimalMsg\022\026\n\016bigdec" +
+      "imal_msg\030\001 \002(\014\"5\n\004UUID\022\026\n\016least_sig_bits" +
+      "\030\001 \002(\004\022\025\n\rmost_sig_bits\030\002 \002(\004\"K\n\023Namespa" +
+      "ceDescriptor\022\014\n\004name\030\001 \002(\014\022&\n\rconfigurat" +
+      "ion\030\002 \003(\0132\017.NameStringPair\"$\n\020RegionServ",
+      "erInfo\022\020\n\010infoPort\030\001 \001(\005*r\n\013CompareType\022" +
+      "\010\n\004LESS\020\000\022\021\n\rLESS_OR_EQUAL\020\001\022\t\n\005EQUAL\020\002\022"
+
+      "\r\n\tNOT_EQUAL\020\003\022\024\n\020GREATER_OR_EQUAL\020\004\022\013\n\007" +
+      "GREATER\020\005\022\t\n\005NO_OP\020\006B>\n*org.apache.hadoo" +
+      "p.hbase.protobuf.generatedB\013HBaseProtosH" +
+      "\001\240\001\001"
     };
     com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
       new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {

http://git-wip-us.apache.org/repos/asf/hbase/blob/60e705b8/hbase-protocol/src/main/protobuf/HBase.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol/src/main/protobuf/HBase.proto b/hbase-protocol/src/main/protobuf/HBase.proto
index 5622735..3e3d570 100644
--- a/hbase-protocol/src/main/protobuf/HBase.proto
+++ b/hbase-protocol/src/main/protobuf/HBase.proto
@@ -159,6 +159,7 @@ message SnapshotDescription {
   enum Type {
     DISABLED = 0;
     FLUSH = 1;
+    SKIPFLUSH = 2;
   }
   optional Type type = 4 [default = FLUSH];
   optional int32 version = 5;

http://git-wip-us.apache.org/repos/asf/hbase/blob/60e705b8/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java
index b0a1b33..468c1af 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java
@@ -48,6 +48,7 @@ public class FlushSnapshotSubprocedure extends Subprocedure {
   private final List<HRegion> regions;
   private final SnapshotDescription snapshot;
   private final SnapshotSubprocedurePool taskManager;
+  private boolean snapshotSkipFlush = false;
 
   public FlushSnapshotSubprocedure(ProcedureMember member,
       ForeignExceptionDispatcher errorListener, long wakeFrequency, long timeout,
@@ -55,6 +56,10 @@ public class FlushSnapshotSubprocedure extends Subprocedure {
       SnapshotSubprocedurePool taskManager) {
     super(member, snapshot.getName(), errorListener, wakeFrequency, timeout);
     this.snapshot = snapshot;
+
+    if (this.snapshot.getType() == SnapshotDescription.Type.SKIPFLUSH) {
+      snapshotSkipFlush = true;
+    }
     this.regions = regions;
     this.taskManager = taskManager;
   }
@@ -78,10 +83,25 @@ public class FlushSnapshotSubprocedure extends Subprocedure {
       LOG.debug("Starting region operation on " + region);
       region.startRegionOperation();
       try {
-        LOG.debug("Flush Snapshotting region " + region.toString() + " started...");
-        region.flushcache();
+        if (snapshotSkipFlush) {
+        /*
+         * This is to take an online-snapshot without force a coordinated flush to prevent
pause
+         * The snapshot type is defined inside the snapshot description. FlushSnapshotSubprocedure
+         * should be renamed to distributedSnapshotSubprocedure, and the flush() behavior
can be
+         * turned on/off based on the flush type.
+         * To minimized the code change, class name is not changed.
+         */
+          LOG.debug("take snapshot without flush memstore first");
+        } else {
+          LOG.debug("Flush Snapshotting region " + region.toString() + " started...");
+          region.flushcache();
+        }
         region.addRegionToSnapshot(snapshot, monitor);
-        LOG.debug("... Flush Snapshotting region " + region.toString() + " completed.");
+        if (snapshotSkipFlush) {
+          LOG.debug("... SkipFlush Snapshotting region " + region.toString() + " completed.");
+        } else {
+          LOG.debug("... Flush Snapshotting region " + region.toString() + " completed.");
+        }
       } finally {
         LOG.debug("Closing region operation on " + region);
         region.closeRegionOperation();

http://git-wip-us.apache.org/repos/asf/hbase/blob/60e705b8/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
index 4a4ee79..e78d690 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
@@ -186,6 +186,19 @@ public class RegionServerSnapshotManager extends RegionServerProcedureManager
{
         new SnapshotSubprocedurePool(rss.getServerName().toString(), conf);
       return new FlushSnapshotSubprocedure(member, exnDispatcher, wakeMillis,
           timeoutMillis, involvedRegions, snapshot, taskManager);
+    case SKIPFLUSH:
+        /*
+         * This is to take an online-snapshot without force a coordinated flush to prevent
pause
+         * The snapshot type is defined inside the snapshot description. FlushSnapshotSubprocedure
+         * should be renamed to distributedSnapshotSubprocedure, and the flush() behavior
can be
+         * turned on/off based on the flush type.
+         * To minimized the code change, class name is not changed.
+         */
+        SnapshotSubprocedurePool taskManager2 =
+            new SnapshotSubprocedurePool(rss.getServerName().toString(), conf);
+        return new FlushSnapshotSubprocedure(member, exnDispatcher, wakeMillis,
+            timeoutMillis, involvedRegions, snapshot, taskManager2);
+
     default:
       throw new UnsupportedOperationException("Unrecognized snapshot type:" + snapshot.getType());
     }

http://git-wip-us.apache.org/repos/asf/hbase/blob/60e705b8/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
index ee12d91..72a6b51 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
@@ -187,6 +187,58 @@ public class TestFlushSnapshotFromClient {
         admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
   }
 
+   /**
+   * Test snapshotting a table that is online without flushing
+   * @throws Exception
+   */
+  @Test
+  public void testSkipFlushTableSnapshot() throws Exception {
+    HBaseAdmin admin = UTIL.getHBaseAdmin();
+    // make sure we don't fail on listing snapshots
+    SnapshotTestingUtils.assertNoSnapshots(admin);
+
+    // put some stuff in the table
+    HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
+    UTIL.loadTable(table, TEST_FAM);
+
+    // get the name of all the regionservers hosting the snapshotted table
+    Set<String> snapshotServers = new HashSet<String>();
+    List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
+    for (RegionServerThread server : servers) {
+      if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
+        snapshotServers.add(server.getRegionServer().getServerName().toString());
+      }
+    }
+
+    LOG.debug("FS state before snapshot:");
+    FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
+        FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
+
+    // take a snapshot of the enabled table
+    String snapshotString = "skipFlushTableSnapshot";
+    byte[] snapshot = Bytes.toBytes(snapshotString);
+    admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.SKIPFLUSH);
+    LOG.debug("Snapshot completed.");
+
+    // make sure we have the snapshot
+    List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
+        snapshot, TABLE_NAME);
+
+    // make sure its a valid snapshot
+    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+    LOG.debug("FS state after snapshot:");
+    FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
+        FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
+
+    SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
+        admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
+
+    admin.deleteSnapshot(snapshot);
+    snapshots = admin.listSnapshots();
+    SnapshotTestingUtils.assertNoSnapshots(admin);
+  }
+
 
   /**
    * Test simple flush snapshotting a table that is online

http://git-wip-us.apache.org/repos/asf/hbase/blob/60e705b8/hbase-shell/src/main/ruby/hbase.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/hbase.rb b/hbase-shell/src/main/ruby/hbase.rb
index 3c09c4d..fcd11fc 100644
--- a/hbase-shell/src/main/ruby/hbase.rb
+++ b/hbase-shell/src/main/ruby/hbase.rb
@@ -61,6 +61,7 @@ module HBaseConstants
   ATTRIBUTES="ATTRIBUTES"
   VISIBILITY="VISIBILITY"
   AUTHORIZATIONS = "AUTHORIZATIONS"
+  SKIP_FLUSH = 'SKIP_FLUSH'
 
   # Load constants from hbase java API
   def self.promote_constants(constants)

http://git-wip-us.apache.org/repos/asf/hbase/blob/60e705b8/hbase-shell/src/main/ruby/hbase/admin.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb
index 9a9ca91..cb70b5c 100644
--- a/hbase-shell/src/main/ruby/hbase/admin.rb
+++ b/hbase-shell/src/main/ruby/hbase/admin.rb
@@ -22,6 +22,7 @@ java_import java.util.Arrays
 java_import org.apache.hadoop.hbase.util.Pair
 java_import org.apache.hadoop.hbase.util.RegionSplitter
 java_import org.apache.hadoop.hbase.util.Bytes
+java_import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos::SnapshotDescription
 
 # Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin
 
@@ -692,8 +693,18 @@ module Hbase
     
     #----------------------------------------------------------------------------------------------
     # Take a snapshot of specified table
-    def snapshot(table, snapshot_name)
-      @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes)
+    def snapshot(table, snapshot_name, *args)
+      if args.empty?
+         @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes)
+      else
+         args.each do |arg|
+            if arg[SKIP_FLUSH] == true
+              @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes, SnapshotDescription::Type::SKIPFLUSH)
+            else
+               @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes)
+            end
+         end
+      end
     end
 
     #----------------------------------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hbase/blob/60e705b8/hbase-shell/src/main/ruby/shell/commands/snapshot.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell/commands/snapshot.rb b/hbase-shell/src/main/ruby/shell/commands/snapshot.rb
index 62de845..15bf298 100644
--- a/hbase-shell/src/main/ruby/shell/commands/snapshot.rb
+++ b/hbase-shell/src/main/ruby/shell/commands/snapshot.rb
@@ -24,13 +24,13 @@ module Shell
 Take a snapshot of specified table. Examples:
 
   hbase> snapshot 'sourceTable', 'snapshotName'
-  hbase> snapshot 'namespace:sourceTable', 'snapshotName'
+  hbase> snapshot 'namespace:sourceTable', 'snapshotName', {SKIP_FLUSH => true}
 EOF
       end
 
-      def command(table, snapshot_name)
+      def command(table, snapshot_name, *args)
         format_simple_command do
-          admin.snapshot(table, snapshot_name)
+          admin.snapshot(table, snapshot_name, *args)
         end
       end
     end


Mime
View raw message