hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-16772 Add verbose option to VerifyReplication for logging good rows
Date Thu, 06 Oct 2016 18:16:27 GMT
Repository: hbase
Updated Branches:
  refs/heads/master 7a3850978 -> 97c133383


HBASE-16772 Add verbose option to VerifyReplication for logging good rows


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/97c13338
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/97c13338
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/97c13338

Branch: refs/heads/master
Commit: 97c13338313b6ccf75f3b7988e97f0db5bf5e177
Parents: 7a38509
Author: tedyu <yuzhihong@gmail.com>
Authored: Thu Oct 6 11:16:18 2016 -0700
Committer: tedyu <yuzhihong@gmail.com>
Committed: Thu Oct 6 11:16:18 2016 -0700

----------------------------------------------------------------------
 .../replication/VerifyReplication.java          | 24 ++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/97c13338/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index 04ae18f..0273b91 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.ConnectionFactory;
@@ -47,8 +46,8 @@ import org.apache.hadoop.hbase.mapreduce.TableMapper;
 import org.apache.hadoop.hbase.mapreduce.TableSplit;
 import org.apache.hadoop.hbase.replication.ReplicationException;
 import org.apache.hadoop.hbase.replication.ReplicationFactory;
-import org.apache.hadoop.hbase.replication.ReplicationPeerZKImpl;
 import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.hadoop.hbase.replication.ReplicationPeerZKImpl;
 import org.apache.hadoop.hbase.replication.ReplicationPeers;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
@@ -86,6 +85,7 @@ public class VerifyReplication extends Configured implements Tool {
   static String peerId = null;
   static String rowPrefixes = null;
   static int sleepMsBeforeReCompare = 0;
+  static boolean verbose = false;
 
   private final static String JOB_NAME_CONF_KEY = "mapreduce.job.name";
 
@@ -107,6 +107,7 @@ public class VerifyReplication extends Configured implements Tool {
     private ResultScanner replicatedScanner;
     private Result currentCompareRowInPeerTable;
     private int sleepMsBeforeReCompare;
+    private boolean verbose = false;
 
     /**
      * Map method that compares every scanned row with the equivalent from
@@ -123,6 +124,7 @@ public class VerifyReplication extends Configured implements Tool {
       if (replicatedScanner == null) {
         Configuration conf = context.getConfiguration();
         sleepMsBeforeReCompare = conf.getInt(NAME +".sleepMsBeforeReCompare", 0);
+        verbose = conf.getBoolean(NAME +".verbose", false);
         final Scan scan = new Scan();
         scan.setBatch(batch);
         scan.setCacheBlocks(false);
@@ -173,6 +175,9 @@ public class VerifyReplication extends Configured implements Tool {
           try {
             Result.compareResults(value, currentCompareRowInPeerTable);
             context.getCounter(Counters.GOODROWS).increment(1);
+            if (verbose) {
+              LOG.info("Good row key: " + delimiter + Bytes.toString(value.getRow()) + delimiter);
+            }
           } catch (Exception e) {
             logFailRowAndIncreaseCounter(context, Counters.CONTENT_DIFFERENT_ROWS, value);
             LOG.error("Exception while comparing row : " + e);
@@ -199,6 +204,10 @@ public class VerifyReplication extends Configured implements Tool {
           Result sourceResult = sourceTable.get(new Get(row.getRow()));
           Result replicatedResult = replicatedTable.get(new Get(row.getRow()));
           Result.compareResults(sourceResult, replicatedResult);
+          context.getCounter(Counters.GOODROWS).increment(1);
+          if (verbose) {
+            LOG.info("Good row key: " + delimiter + Bytes.toString(row.getRow()) + delimiter);
+          }
           return;
         } catch (Exception e) {
           LOG.error("recompare fail after sleep, rowkey=" + delimiter +
@@ -311,6 +320,7 @@ public class VerifyReplication extends Configured implements Tool {
     conf.setLong(NAME+".startTime", startTime);
     conf.setLong(NAME+".endTime", endTime);
     conf.setInt(NAME +".sleepMsBeforeReCompare", sleepMsBeforeReCompare);
+    conf.setBoolean(NAME +".verbose", verbose);
     if (families != null) {
       conf.set(NAME+".families", families);
     }
@@ -451,7 +461,11 @@ public class VerifyReplication extends Configured implements Tool {
           sleepMsBeforeReCompare = Integer.parseInt(cmd.substring(sleepToReCompareKey.length()));
           continue;
         }
-        
+        final String verboseKey = "--verbose";
+        if (cmd.startsWith(verboseKey)) {
+          verbose = true;
+          continue;
+        }        
         if (i == args.length-2) {
           peerId = cmd;
         }
@@ -487,7 +501,8 @@ public class VerifyReplication extends Configured implements Tool {
       System.err.println("ERROR: " + errorMsg);
     }
     System.err.println("Usage: verifyrep [--starttime=X]" +
-        " [--endtime=Y] [--families=A] [--row-prefixes=B] [--delimiter=] <peerid> <tablename>");
+        " [--endtime=Y] [--families=A] [--row-prefixes=B] [--delimiter=] [--recomparesleep=]
" +
+        "[--verbose] <peerid> <tablename>");
     System.err.println();
     System.err.println("Options:");
     System.err.println(" starttime    beginning of the time range");
@@ -499,6 +514,7 @@ public class VerifyReplication extends Configured implements Tool {
     System.err.println(" delimiter    the delimiter used in display around rowkey");
     System.err.println(" recomparesleep   milliseconds to sleep before recompare row, " +
         "default value is 0 which disables the recompare.");
+    System.err.println(" verbose      logs row keys of good rows");
     System.err.println();
     System.err.println("Args:");
     System.err.println(" peerid       Id of the peer used for verification, must match the
one given for replication");


Mime
View raw message