hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From syuanji...@apache.org
Subject [42/50] [abbrv] hbase git commit: HBASE-15191 CopyTable and VerifyReplication - Option to specify batch size, versions (Parth Shah)
Date Wed, 30 Mar 2016 16:03:44 GMT
HBASE-15191 CopyTable and VerifyReplication - Option to specify batch size, versions (Parth
Shah)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cbf9c1e1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cbf9c1e1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cbf9c1e1

Branch: refs/heads/hbase-12439
Commit: cbf9c1e116717972c243abedb53e7d26ef03cc85
Parents: 7793bc5
Author: tedyu <yuzhihong@gmail.com>
Authored: Mon Mar 28 17:25:00 2016 -0700
Committer: tedyu <yuzhihong@gmail.com>
Committed: Mon Mar 28 17:25:00 2016 -0700

----------------------------------------------------------------------
 .../hadoop/hbase/mapreduce/CopyTable.java       | 31 ++++++++++++++++----
 .../replication/VerifyReplication.java          |  9 ++++++
 2 files changed, 35 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/cbf9c1e1/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
index b7f4b66..dd8b891 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
@@ -54,7 +54,9 @@ public class CopyTable extends Configured implements Tool {
 
   final static String NAME = "copytable";
   long startTime = 0;
-  long endTime = 0;
+  long endTime = HConstants.LATEST_TIMESTAMP;
+  int batch = Integer.MAX_VALUE;
+  int cacheRow = -1;
   int versions = -1;
   String tableName = null;
   String startRow = null;
@@ -82,15 +84,22 @@ public class CopyTable extends Configured implements Tool {
     if (!doCommandLine(args)) {
       return null;
     }
-
+    
     Job job = Job.getInstance(getConf(), getConf().get(JOB_NAME_CONF_KEY, NAME + "_" + tableName));
     job.setJarByClass(CopyTable.class);
     Scan scan = new Scan();
+    
+    scan.setBatch(batch);
     scan.setCacheBlocks(false);
-    if (startTime != 0) {
-      scan.setTimeRange(startTime,
-          endTime == 0 ? HConstants.LATEST_TIMESTAMP : endTime);
+    
+    if (cacheRow > 0) {
+      scan.setCaching(cacheRow);
+    } else {
+      scan.setCaching(getConf().getInt(HConstants.HBASE_CLIENT_SCANNER_CACHING, 100));
     }
+    
+    scan.setTimeRange(startTime, endTime);
+    
     if (allCells) {
       scan.setRaw(true);
     }
@@ -252,6 +261,18 @@ public class CopyTable extends Configured implements Tool {
           endTime = Long.parseLong(cmd.substring(endTimeArgKey.length()));
           continue;
         }
+        
+        final String batchArgKey = "--batch=";
+        if (cmd.startsWith(batchArgKey)) {
+          batch = Integer.parseInt(cmd.substring(batchArgKey.length()));
+          continue;
+        }
+        
+        final String cacheRowArgKey = "--cacheRow=";
+        if (cmd.startsWith(cacheRowArgKey)) {
+          cacheRow = Integer.parseInt(cmd.substring(cacheRowArgKey.length()));
+          continue;
+        }
 
         final String versionsArgKey = "--versions=";
         if (cmd.startsWith(versionsArgKey)) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/cbf9c1e1/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index e6b4802..a452036 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -72,6 +72,7 @@ public class VerifyReplication extends Configured implements Tool {
   private final static String PEER_CONFIG_PREFIX = NAME + ".peer.";
   static long startTime = 0;
   static long endTime = Long.MAX_VALUE;
+  static int batch = Integer.MAX_VALUE;
   static int versions = -1;
   static String tableName = null;
   static String families = null;
@@ -110,6 +111,8 @@ public class VerifyReplication extends Configured implements Tool {
       if (replicatedScanner == null) {
         Configuration conf = context.getConfiguration();
         final Scan scan = new Scan();
+        scan.setBatch(batch);
+        scan.setCacheBlocks(false);
         scan.setCaching(conf.getInt(TableInputFormat.SCAN_CACHEDROWS, 1));
         long startTime = conf.getLong(NAME + ".startTime", 0);
         long endTime = conf.getLong(NAME + ".endTime", Long.MAX_VALUE);
@@ -338,6 +341,12 @@ public class VerifyReplication extends Configured implements Tool {
           versions = Integer.parseInt(cmd.substring(versionsArgKey.length()));
           continue;
         }
+        
+        final String batchArgKey = "--batch=";
+        if (cmd.startsWith(batchArgKey)) {
+          batch = Integer.parseInt(cmd.substring(batchArgKey.length()));
+          continue;
+        }
 
         final String familiesArgKey = "--families=";
         if (cmd.startsWith(familiesArgKey)) {


Mime
View raw message