hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject hbase git commit: HBASE-21171 [amv2] Tool to parse a directory of MasterProcWALs standalone
Date Sun, 09 Sep 2018 16:29:12 GMT
Repository: hbase
Updated Branches:
  refs/heads/master a51c33385 -> b83613fdc


HBASE-21171 [amv2] Tool to parse a directory of MasterProcWALs standalone

Signed-off-by: Mike Drob <mdrob@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b83613fd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b83613fd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b83613fd

Branch: refs/heads/master
Commit: b83613fdce7c5e57766417768b2fa1a13fcb4106
Parents: a51c333
Author: Michael Stack <stack@apache.org>
Authored: Fri Sep 7 17:07:11 2018 -0700
Committer: Michael Stack <stack@apache.org>
Committed: Sun Sep 9 09:28:59 2018 -0700

----------------------------------------------------------------------
 .../store/wal/ProcedureWALPrettyPrinter.java    |  1 +
 .../procedure2/store/wal/WALProcedureStore.java | 38 ++++++++++++++++----
 2 files changed, 33 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/b83613fd/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPrettyPrinter.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPrettyPrinter.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPrettyPrinter.java
index b74efe4..582db77 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPrettyPrinter.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPrettyPrinter.java
@@ -49,6 +49,7 @@ import org.apache.hbase.thirdparty.org.apache.commons.cli.PosixParser;
 
 /**
  * ProcedureWALPrettyPrinter prints the contents of a given ProcedureWAL file
+ * @see WALProcedureStore#main(String[]) if you want to check parse of a directory of WALs.
  */
 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
 @InterfaceStability.Evolving

http://git-wip-us.apache.org/repos/asf/hbase/blob/b83613fd/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
index b99b1c1..8e7b2cc 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
@@ -43,9 +43,11 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.log.HBaseMarkers;
 import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
 import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreBase;
 import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker;
 import org.apache.hadoop.hbase.procedure2.util.ByteSlot;
@@ -64,6 +66,8 @@ import org.apache.hbase.thirdparty.org.apache.commons.collections4.queue.Circula
 
 /**
  * WAL implementation of the ProcedureStore.
+ * @see ProcedureWALPrettyPrinter for printing content of a single WAL.
+ * @see #main(String[]) to parse a directory of MasterWALProcs.
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
@@ -424,7 +428,6 @@ public class WALProcedureStore extends ProcedureStoreBase {
       it.next(); // Skip the current log
 
       ProcedureWALFormat.load(it, storeTracker, new ProcedureWALFormat.Loader() {
-        long count = 0;
 
         @Override
         public void setMaxProcId(long maxProcId) {
@@ -433,11 +436,6 @@ public class WALProcedureStore extends ProcedureStoreBase {
 
         @Override
         public void load(ProcedureIterator procIter) throws IOException {
-          if ((++count % 1000) == 0) {
-            // Log every 1000 procedures otherwise it looks like Master is dead if loads
of WALs
-            // and procedures to load.
-            LOG.debug("Loaded {} procedures", this.count);
-          }
           loader.load(procIter);
         }
 
@@ -1324,4 +1322,32 @@ public class WALProcedureStore extends ProcedureStoreBase {
     log.close();
     return log;
   }
+
+  /**
+   * Parses a directory of WALs building up ProcedureState.
+   * For testing parse and profiling.
+   * @param args Include pointer to directory of WAL files for a store instance to parse
& load.
+   */
+  public static void main(String [] args) throws IOException {
+    Configuration conf = HBaseConfiguration.create();
+    if (args == null || args.length != 1) {
+      System.out.println("ERROR: Empty arguments list; pass path to MASTERPROCWALS_DIR.");
+      System.out.println("Usage: WALProcedureStore MASTERPROCWALS_DIR");
+      System.exit(-1);
+    }
+    WALProcedureStore store = new WALProcedureStore(conf, new Path(args[0]), null,
+      new WALProcedureStore.LeaseRecovery() {
+        @Override
+        public void recoverFileLease(FileSystem fs, Path path) throws IOException {
+          // no-op
+        }
+      });
+    try {
+      store.start(16);
+      ProcedureExecutor pe = new ProcedureExecutor(conf, new Object()/*Pass anything*/, store);
+      pe.init(1, true);
+    } finally {
+      store.stop(true);
+    }
+  }
 }


Mime
View raw message