hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject [4/4] hbase git commit: HBASE-16946 Provide Raw scan as an option in VerifyReplication (Sreekar Pallapothu)
Date Sat, 03 Dec 2016 00:06:46 GMT
HBASE-16946 Provide Raw scan as an option in VerifyReplication (Sreekar Pallapothu)

Conflicts:
	hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java

Amending-Author: Andrew Purtell <apurtell@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4cdd0adf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4cdd0adf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4cdd0adf

Branch: refs/heads/0.98
Commit: 4cdd0adfbdfb8b4c28b7e226b88fb69d0e4e404f
Parents: e2df4b8
Author: tedyu <yuzhihong@gmail.com>
Authored: Fri Nov 4 03:36:17 2016 -0700
Committer: Andrew Purtell <apurtell@apache.org>
Committed: Fri Dec 2 16:04:29 2016 -0800

----------------------------------------------------------------------
 .../replication/VerifyReplication.java          |  13 +++
 .../replication/TestReplicationSmallTests.java  | 102 ++++++++++++++++++-
 2 files changed, 112 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/4cdd0adf/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index 1ec31a1..b22b11e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -84,6 +84,7 @@ public class VerifyReplication extends Configured implements Tool {
   static String peerId = null;
   static String rowPrefixes = null;
   static boolean verbose = false;
+  static boolean includeDeletedCells = false;
 
   /**
    * Map-only comparator for 2 tables
@@ -126,6 +127,8 @@ public class VerifyReplication extends Configured implements Tool {
             scan.addFamily(Bytes.toBytes(fam));
           }
         }
+        boolean includeDeletedCells = conf.getBoolean(NAME + ".includeDeletedCells", false);
+        scan.setRaw(includeDeletedCells);
         String rowPrefixes = conf.get(NAME + ".rowPrefixes", null);
         setRowPrefixFilter(scan, rowPrefixes);
         scan.setTimeRange(startTime, endTime);
@@ -267,6 +270,7 @@ public class VerifyReplication extends Configured implements Tool {
     conf.setLong(NAME+".startTime", startTime);
     conf.setLong(NAME+".endTime", endTime);
     conf.setBoolean(NAME +".verbose", verbose);
+    conf.setBoolean(NAME +".includeDeletedCells", includeDeletedCells);
     if (families != null) {
       conf.set(NAME+".families", families);
     }
@@ -291,6 +295,7 @@ public class VerifyReplication extends Configured implements Tool {
 
     Scan scan = new Scan();
     scan.setTimeRange(startTime, endTime);
+    scan.setRaw(includeDeletedCells);
     if (versions >= 0) {
       scan.setMaxVersions(versions);
       LOG.info("Number of versions set to " + versions);
@@ -368,6 +373,12 @@ public class VerifyReplication extends Configured implements Tool {
           continue;
         }
 
+        final String includeDeletedCellsArgKey = "--raw";
+        if (cmd.equals(includeDeletedCellsArgKey)) {
+          includeDeletedCells = true;
+          continue;
+        }
+
         final String versionsArgKey = "--versions=";
         if (cmd.startsWith(versionsArgKey)) {
           versions = Integer.parseInt(cmd.substring(versionsArgKey.length()));
@@ -433,6 +444,7 @@ public class VerifyReplication extends Configured implements Tool {
     families = null;
     peerId = null;
     rowPrefixes = null;
+    includeDeletedCells = false;
   }
 
   /*
@@ -451,6 +463,7 @@ public class VerifyReplication extends Configured implements Tool {
     System.err.println("              without endtime means from starttime to forever");
     System.err.println(" endtime      end of the time range");
     System.err.println(" versions     number of cell versions to verify");
+    System.err.println(" raw          includes raw scan if given in options");
     System.err.println(" families     comma-separated list of families to copy");
     System.err.println(" row-prefixes comma-separated list of row key prefixes to filter
on ");
     System.err.println(" delimiter    the delimiter used in display around rowkey");

http://git-wip-us.apache.org/repos/asf/hbase/blob/4cdd0adf/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
index 5df28a5..6ddc209 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
@@ -33,18 +33,18 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.ServerLoad;
-import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
@@ -486,6 +486,102 @@ public class TestReplicationSmallTests extends TestReplicationBase {
     runVerifyReplication(args, 0, NB_ROWS_IN_BATCH);
   }
 
+  /**
+   * Load a row into a table, make sure the data is really the same,
+   * delete the row, make sure the delete marker is replicated,
+   * run verify replication with and without raw to check the results.
+   * @throws Exception
+   */
+  @Test(timeout=300000)
+  public void testVerifyRepJobWithRawOptions() throws Exception {
+    LOG.info("testVerifyRepJobWithRawOptions");
+
+    TableName tablename = TableName.valueOf("test_raw");
+    byte[] familyname = Bytes.toBytes("fam_raw");
+    byte[] row = Bytes.toBytes("row_raw");
+
+    HTableInterface lHtable1 = null;
+    HTableInterface lHtable2 = null;
+
+    try {
+      HTableDescriptor table = new HTableDescriptor(tablename);
+      HColumnDescriptor fam = new HColumnDescriptor(familyname);
+      fam.setMaxVersions(100);
+      fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
+      table.addFamily(fam);
+
+      HBaseAdmin admin1 = new HBaseAdmin(conf1);
+      try {
+        admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
+      } finally {
+        admin1.close();
+      }
+      utility1.waitUntilAllRegionsAssigned(tablename);
+
+      HBaseAdmin admin2 = new HBaseAdmin(conf2);
+      try {
+        admin2.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
+      } finally {
+        admin2.close();
+      }
+      utility2.waitUntilAllRegionsAssigned(tablename);
+
+      lHtable1 = HConnectionManager.createConnection(conf1).getTable(tablename);
+      lHtable2 = HConnectionManager.createConnection(conf2).getTable(tablename);
+
+      Put put = new Put(row);
+      put.add(familyname, row, row);
+      lHtable1.put(put);
+
+      Get get = new Get(row);
+      for (int i = 0; i < NB_RETRIES; i++) {
+        if (i==NB_RETRIES-1) {
+          fail("Waited too much time for put replication");
+        }
+        Result res = lHtable2.get(get);
+        if (res.size() == 0) {
+          LOG.info("Row not available");
+          Thread.sleep(SLEEP_TIME);
+        } else {
+          assertArrayEquals(res.value(), row);
+          break;
+        }
+      }
+
+      Delete del = new Delete(row);
+      lHtable1.delete(del);
+
+      get = new Get(row);
+      for (int i = 0; i < NB_RETRIES; i++) {
+        if (i==NB_RETRIES-1) {
+          fail("Waited too much time for del replication");
+        }
+        Result res = lHtable2.get(get);
+        if (res.size() >= 1) {
+          LOG.info("Row not deleted");
+          Thread.sleep(SLEEP_TIME);
+        } else {
+          break;
+        }
+      }
+
+      // Checking verifyReplication for the default behavior.
+      String[] argsWithoutRaw = new String[] {PEER_ID, tablename.getNameAsString()};
+      runVerifyReplication(argsWithoutRaw, 0, 0);
+
+      // Checking verifyReplication with raw
+      String[] argsWithRawAsTrue = new String[] {"--raw", PEER_ID, tablename.getNameAsString()};
+      runVerifyReplication(argsWithRawAsTrue, 1, 0);
+    } finally {
+      if (lHtable1 != null) {
+        lHtable1.close();
+      }
+      if (lHtable2 != null) {
+        lHtable2.close();
+      }
+    }
+  }
+
   private void runVerifyReplication(String[] args, int expectedGoodRows, int expectedBadRows)
       throws IOException, InterruptedException, ClassNotFoundException {
     Job job = VerifyReplication.createSubmittableJob(new Configuration(CONF_WITH_LOCALFS),
args);


Mime
View raw message