hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From anoopsamj...@apache.org
Subject hbase git commit: HBASE-15228 Add the methods to RegionObserver to trigger start/complete restoring WALs. (Toshihiro Suzuki)
Date Sat, 21 May 2016 04:26:49 GMT
Repository: hbase
Updated Branches:
  refs/heads/master b1bf622e6 -> 0e52ac246


HBASE-15228 Add the methods to RegionObserver to trigger start/complete restoring WALs. (Toshihiro
Suzuki)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0e52ac24
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0e52ac24
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0e52ac24

Branch: refs/heads/master
Commit: 0e52ac2464afe049b14930eeb3e879698743f27e
Parents: b1bf622
Author: anoopsjohn <anoopsamjohn@gmail.com>
Authored: Sat May 21 09:56:28 2016 +0530
Committer: anoopsjohn <anoopsamjohn@gmail.com>
Committed: Sat May 21 09:56:28 2016 +0530

----------------------------------------------------------------------
 .../hbase/coprocessor/BaseRegionObserver.java   | 10 ++++++
 .../hbase/coprocessor/RegionObserver.java       | 22 ++++++++++++
 .../hadoop/hbase/regionserver/HRegion.java      |  8 +++++
 .../regionserver/RegionCoprocessorHost.java     | 30 ++++++++++++++++
 .../hbase/coprocessor/SimpleRegionObserver.java | 30 ++++++++++++++++
 .../TestRegionObserverInterface.java            | 36 ++++++++++++--------
 6 files changed, 122 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/0e52ac24/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
index eaf9f43..bff727a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
@@ -452,6 +452,16 @@ public class BaseRegionObserver implements RegionObserver {
       final InternalScanner s) throws IOException {
   }
 
+  @Override
+  public void preReplayWALs(ObserverContext<? extends RegionCoprocessorEnvironment>
env,
+      HRegionInfo info, Path edits) throws IOException {
+  }
+
+  @Override
+  public void postReplayWALs(ObserverContext<? extends RegionCoprocessorEnvironment>
env,
+      HRegionInfo info, Path edits) throws IOException {
+  }
+
   /**
    * Implementers should override this version of the method and leave the deprecated one
as-is.
    */

http://git-wip-us.apache.org/repos/asf/hbase/blob/0e52ac24/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
index 1d985df..4729954 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
@@ -1219,6 +1219,28 @@ public interface RegionObserver extends Coprocessor {
     throws IOException;
 
   /**
+   * Called before replaying WALs for this region.
+   * Calling {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} has no
+   * effect in this hook.
+   * @param ctx the environment provided by the region server
+   * @param info the RegionInfo for this region
+   * @param edits the file of recovered edits
+   * @throws IOException if an error occurred on the coprocessor
+   */
+  void preReplayWALs(final ObserverContext<? extends RegionCoprocessorEnvironment>
ctx,
+      HRegionInfo info, Path edits) throws IOException;
+
+  /**
+   * Called after replaying WALs for this region.
+   * @param ctx the environment provided by the region server
+   * @param info the RegionInfo for this region
+   * @param edits the file of recovered edits
+   * @throws IOException if an error occurred on the coprocessor
+   */
+  void postReplayWALs(final ObserverContext<? extends RegionCoprocessorEnvironment>
ctx,
+      HRegionInfo info, Path edits) throws IOException;
+
+  /**
    * Called before a {@link org.apache.hadoop.hbase.regionserver.wal.WALEdit}
    * replayed for this region.
    */

http://git-wip-us.apache.org/repos/asf/hbase/blob/0e52ac24/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index e5f9d50..16d4e93 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -4026,6 +4026,10 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver,
Regi
         int period = this.conf.getInt("hbase.hstore.report.period", 300000);
         long lastReport = EnvironmentEdgeManager.currentTime();
 
+        if (coprocessorHost != null) {
+          coprocessorHost.preReplayWALs(this.getRegionInfo(), edits);
+        }
+
         while ((entry = reader.next()) != null) {
           WALKey key = entry.getKey();
           WALEdit val = entry.getEdit();
@@ -4144,6 +4148,10 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver,
Regi
             coprocessorHost.postWALRestore(this.getRegionInfo(), key, val);
           }
         }
+
+        if (coprocessorHost != null) {
+          coprocessorHost.postReplayWALs(this.getRegionInfo(), edits);
+        }
       } catch (EOFException eof) {
         Path p = WALSplitter.moveAsideBadEditsFile(fs, edits);
         msg = "Encountered EOF. Most likely due to Master failure during " +

http://git-wip-us.apache.org/repos/asf/hbase/blob/0e52ac24/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index 25e0200..f6ccaa1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -1402,6 +1402,36 @@ public class RegionCoprocessorHost
   }
 
   /**
+   * @param info the RegionInfo for this region
+   * @param edits the file of recovered edits
+   * @throws IOException Exception
+   */
+  public void preReplayWALs(final HRegionInfo info, final Path edits) throws IOException
{
+    execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
+      @Override
+      public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment>
ctx)
+        throws IOException {
+        oserver.preReplayWALs(ctx, info, edits);
+      }
+    });
+  }
+
+  /**
+   * @param info the RegionInfo for this region
+   * @param edits the file of recovered edits
+   * @throws IOException Exception
+   */
+  public void postReplayWALs(final HRegionInfo info, final Path edits) throws IOException
{
+    execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
+      @Override
+      public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment>
ctx)
+        throws IOException {
+        oserver.postReplayWALs(ctx, info, edits);
+      }
+    });
+  }
+
+  /**
    * @param info
    * @param logKey
    * @param logEdit

http://git-wip-us.apache.org/repos/asf/hbase/blob/0e52ac24/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
index 9e50e4e..dfd41a8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
@@ -126,6 +126,8 @@ public class SimpleRegionObserver extends BaseRegionObserver {
   final AtomicInteger ctPostBulkLoadHFile = new AtomicInteger(0);
   final AtomicInteger ctPreBatchMutate = new AtomicInteger(0);
   final AtomicInteger ctPostBatchMutate = new AtomicInteger(0);
+  final AtomicInteger ctPreReplayWALs = new AtomicInteger(0);
+  final AtomicInteger ctPostReplayWALs = new AtomicInteger(0);
   final AtomicInteger ctPreWALRestore = new AtomicInteger(0);
   final AtomicInteger ctPostWALRestore = new AtomicInteger(0);
   final AtomicInteger ctPreWALRestoreDeprecated = new AtomicInteger(0);
@@ -651,6 +653,18 @@ public class SimpleRegionObserver extends BaseRegionObserver {
   }
 
   @Override
+  public void preReplayWALs(ObserverContext<? extends RegionCoprocessorEnvironment>
env,
+      HRegionInfo info, Path edits) throws IOException {
+    ctPreReplayWALs.incrementAndGet();
+  }
+
+  @Override
+  public void postReplayWALs(ObserverContext<? extends RegionCoprocessorEnvironment>
env,
+      HRegionInfo info, Path edits) throws IOException {
+    ctPostReplayWALs.incrementAndGet();
+  }
+
+  @Override
   public void preWALRestore(ObserverContext<? extends RegionCoprocessorEnvironment>
env,
       HRegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException {
     String tableName = logKey.getTablename().getNameAsString();
@@ -798,6 +812,14 @@ public class SimpleRegionObserver extends BaseRegionObserver {
     return ctPrePrepareDeleteTS.get() > 0;
   }
 
+  public boolean hadPreReplayWALs() {
+    return ctPreReplayWALs.get() > 0;
+  }
+
+  public boolean hadPostReplayWALs() {
+    return ctPostReplayWALs.get() > 0;
+  }
+
   public boolean hadPreWALRestore() {
     return ctPreWALRestore.get() > 0;
   }
@@ -931,6 +953,14 @@ public class SimpleRegionObserver extends BaseRegionObserver {
     return ctPostIncrement.get();
   }
 
+  public int getCtPreReplayWALs() {
+    return ctPreReplayWALs.get();
+  }
+
+  public int getCtPostReplayWALs() {
+    return ctPostReplayWALs.get();
+  }
+
   public int getCtPreWALRestore() {
     return ctPreWALRestore.get();
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/0e52ac24/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
index abfadec..5446570 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java
@@ -607,7 +607,6 @@ public class TestRegionObserverInterface {
     }
   }
 
-  @Ignore // TODO: HBASE-13391 to fix flaky test
   @Test (timeout=300000)
   public void testRecovery() throws Exception {
     LOG.info(TestRegionObserverInterface.class.getName() +".testRecovery");
@@ -630,6 +629,9 @@ public class TestRegionObserverInterface {
       put.addColumn(C, C, C);
       table.put(put);
 
+      // put two times
+      table.put(put);
+
       verifyMethodResult(SimpleRegionObserver.class,
           new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
         "hadPreBatchMutate", "hadPostBatchMutate", "hadDelete"},
@@ -638,10 +640,11 @@ public class TestRegionObserverInterface {
           );
 
       verifyMethodResult(SimpleRegionObserver.class,
-          new String[] {"getCtPreWALRestore", "getCtPostWALRestore", "getCtPrePut", "getCtPostPut",
-              "getCtPreWALRestoreDeprecated", "getCtPostWALRestoreDeprecated"},
+          new String[] {"getCtPreReplayWALs", "getCtPostReplayWALs", "getCtPreWALRestore",
+            "getCtPostWALRestore", "getCtPrePut", "getCtPostPut",
+            "getCtPreWALRestoreDeprecated", "getCtPostWALRestoreDeprecated"},
           tableName,
-          new Integer[] {0, 0, 1, 1, 0, 0});
+          new Integer[] {0, 0, 0, 0, 2, 2, 0, 0});
 
       cluster.killRegionServer(rs1.getRegionServer().getServerName());
       Threads.sleep(1000); // Let the kill soak in.
@@ -649,17 +652,17 @@ public class TestRegionObserverInterface {
       LOG.info("All regions assigned");
 
       verifyMethodResult(SimpleRegionObserver.class,
-          new String[] {"getCtPreWALRestore", "getCtPostWALRestore", "getCtPrePut", "getCtPostPut",
-              "getCtPreWALRestoreDeprecated", "getCtPostWALRestoreDeprecated"},
+          new String[] {"getCtPreReplayWALs", "getCtPostReplayWALs", "getCtPreWALRestore",
+            "getCtPostWALRestore", "getCtPrePut", "getCtPostPut",
+            "getCtPreWALRestoreDeprecated", "getCtPostWALRestoreDeprecated"},
           tableName,
-          new Integer[]{1, 1, 0, 0, 0, 0});
+          new Integer[]{1, 1, 2, 2, 0, 0, 0, 0});
     } finally {
       util.deleteTable(tableName);
       table.close();
     }
   }
 
-  @Ignore // TODO: HBASE-13391 to fix flaky test
   @Test (timeout=300000)
   public void testLegacyRecovery() throws Exception {
     LOG.info(TestRegionObserverInterface.class.getName() +".testLegacyRecovery");
@@ -682,6 +685,9 @@ public class TestRegionObserverInterface {
         put.addColumn(C, C, C);
         table.put(put);
 
+        // put two times
+        table.put(put);
+
         verifyMethodResult(SimpleRegionObserver.Legacy.class,
             new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
                 "hadPreBatchMutate", "hadPostBatchMutate", "hadDelete"},
@@ -690,10 +696,11 @@ public class TestRegionObserverInterface {
         );
 
         verifyMethodResult(SimpleRegionObserver.Legacy.class,
-            new String[] {"getCtPreWALRestore", "getCtPostWALRestore", "getCtPrePut",
-                "getCtPostPut", "getCtPreWALRestoreDeprecated", "getCtPostWALRestoreDeprecated"},
+            new String[] {"getCtPreReplayWALs", "getCtPostReplayWALs", "getCtPreWALRestore",
+              "getCtPostWALRestore", "getCtPrePut", "getCtPostPut",
+              "getCtPreWALRestoreDeprecated", "getCtPostWALRestoreDeprecated"},
             tableName,
-            new Integer[] {0, 0, 1, 1, 0, 0});
+            new Integer[] {0, 0, 0, 0, 2, 2, 0, 0});
 
         cluster.killRegionServer(rs1.getRegionServer().getServerName());
         Threads.sleep(1000); // Let the kill soak in.
@@ -701,10 +708,11 @@ public class TestRegionObserverInterface {
         LOG.info("All regions assigned");
 
         verifyMethodResult(SimpleRegionObserver.Legacy.class,
-            new String[] {"getCtPreWALRestore", "getCtPostWALRestore", "getCtPrePut",
-                "getCtPostPut", "getCtPreWALRestoreDeprecated", "getCtPostWALRestoreDeprecated"},
+            new String[] {"getCtPreReplayWALs", "getCtPostReplayWALs", "getCtPreWALRestore",
+              "getCtPostWALRestore", "getCtPrePut", "getCtPostPut",
+              "getCtPreWALRestoreDeprecated", "getCtPostWALRestoreDeprecated"},
             tableName,
-            new Integer[]{1, 1, 0, 0, 1, 1});
+            new Integer[]{1, 1, 2, 2, 0, 0, 2, 2});
       }
     } finally {
       util.deleteTable(tableName);


Mime
View raw message