hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-17826 Backup: submit M/R job to a particular Yarn queue (Vladimir Rodionov)
Date Wed, 30 Aug 2017 23:11:45 GMT
Repository: hbase
Updated Branches:
  refs/heads/master b5942241c -> 645e5a5e2


HBASE-17826 Backup: submit M/R job to a particular Yarn queue (Vladimir Rodionov)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/645e5a5e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/645e5a5e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/645e5a5e

Branch: refs/heads/master
Commit: 645e5a5e2b598310513db7ccfa559f8961c65f05
Parents: b594224
Author: tedyu <yuzhihong@gmail.com>
Authored: Wed Aug 30 16:11:32 2017 -0700
Committer: tedyu <yuzhihong@gmail.com>
Committed: Wed Aug 30 16:11:32 2017 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hbase/backup/BackupRequest.java | 14 ++++++++++++++
 .../hadoop/hbase/backup/BackupRestoreConstants.java   |  4 ++++
 .../org/apache/hadoop/hbase/backup/RestoreDriver.java | 12 +++++++++++-
 .../hadoop/hbase/backup/impl/BackupCommands.java      | 10 ++++++++++
 4 files changed, 39 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/645e5a5e/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java
----------------------------------------------------------------------
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java
b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java
index dadd861..8659868 100644
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java
+++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java
@@ -67,6 +67,11 @@ public final class BackupRequest {
       return this;
     }
 
+    public Builder withYarnPoolName(String name) {
+      request.setYarnPoolName(name);
+      return this;
+    }
+
     public BackupRequest build() {
       return request;
     }
@@ -79,6 +84,7 @@ public final class BackupRequest {
   private int totalTasks = -1;
   private long bandwidth = -1L;
   private String backupSetName;
+  private String yarnPoolName;
 
   private BackupRequest() {
   }
@@ -136,4 +142,12 @@ public final class BackupRequest {
     this.backupSetName = backupSetName;
     return this;
   }
+
+  public String getYarnPoolName() {
+    return yarnPoolName;
+  }
+
+  public void setYarnPoolName(String yarnPoolName) {
+    this.yarnPoolName = yarnPoolName;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/645e5a5e/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
----------------------------------------------------------------------
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
index 48e70a1..f511cc4 100644
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
+++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
@@ -89,6 +89,10 @@ public interface BackupRestoreConstants {
   public static final String OPTION_TABLE_MAPPING_DESC =
       "A comma separated list of target tables. "
           + "If specified, each table in <tables> must have a mapping";
+  public static final String OPTION_YARN_QUEUE_NAME = "q";
+  public static final String OPTION_YARN_QUEUE_NAME_DESC = "Yarn queue name to run backup
create command on";
+  public static final String OPTION_YARN_QUEUE_NAME_RESTORE_DESC = "Yarn queue name to run
backup restore command on";
+
 
   public static final String BACKUP_CONFIG_STRING =  BackupRestoreConstants.BACKUP_ENABLE_KEY
+ "=true\n"
       + "hbase.master.logcleaner.plugins="

http://git-wip-us.apache.org/repos/asf/hbase/blob/645e5a5e/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java
----------------------------------------------------------------------
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java
b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java
index 82a1b56..2a1458c 100644
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java
+++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java
@@ -29,6 +29,8 @@ import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE
 import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_LIST_DESC;
 import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_MAPPING;
 import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_MAPPING_DESC;
+import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME;
+import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME_RESTORE_DESC;
 
 import java.io.IOException;
 import java.net.URI;
@@ -125,6 +127,13 @@ public class RestoreDriver extends AbstractHBaseTool {
       printToolUsage();
       return -1;
     }
+
+    if (cmd.hasOption(OPTION_YARN_QUEUE_NAME)) {
+      String queueName = cmd.getOptionValue(OPTION_YARN_QUEUE_NAME);
+      // Set system property value for MR job
+      System.setProperty("mapreduce.job.queuename", queueName);
+    }
+
     // parse main restore command options
     String[] remainArgs = cmd.getArgs();
     if (remainArgs.length != 2) {
@@ -195,8 +204,9 @@ public class RestoreDriver extends AbstractHBaseTool {
     addOptNoArg(OPTION_DEBUG, OPTION_DEBUG_DESC);
     addOptWithArg(OPTION_SET, OPTION_SET_RESTORE_DESC);
     addOptWithArg(OPTION_TABLE, OPTION_TABLE_LIST_DESC);
-
     addOptWithArg(OPTION_TABLE_MAPPING, OPTION_TABLE_MAPPING_DESC);
+    addOptWithArg(OPTION_YARN_QUEUE_NAME, OPTION_YARN_QUEUE_NAME_RESTORE_DESC);
+
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hbase/blob/645e5a5e/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
----------------------------------------------------------------------
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
index 650ba2e..eaba049 100644
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
+++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
@@ -32,6 +32,8 @@ import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE
 import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_LIST_DESC;
 import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS;
 import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS_DESC;
+import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME;
+import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME_DESC;
 
 import java.io.IOException;
 import java.net.URI;
@@ -321,6 +323,12 @@ public final class BackupCommands {
           cmdline.hasOption(OPTION_WORKERS) ? Integer.parseInt(cmdline
               .getOptionValue(OPTION_WORKERS)) : -1;
 
+      if (cmdline.hasOption(OPTION_YARN_QUEUE_NAME)) {
+        String poolName = cmdline.getOptionValue(OPTION_YARN_QUEUE_NAME);
+        // Set system property value for MR job
+        System.setProperty("mapreduce.job.queuename", poolName);
+      }
+
       try (BackupAdminImpl admin = new BackupAdminImpl(conn);) {
 
         BackupRequest.Builder builder = new BackupRequest.Builder();
@@ -368,6 +376,8 @@ public final class BackupCommands {
       options.addOption(OPTION_BANDWIDTH, true, OPTION_BANDWIDTH_DESC);
       options.addOption(OPTION_SET, true, OPTION_SET_BACKUP_DESC);
       options.addOption(OPTION_TABLE, true, OPTION_TABLE_LIST_DESC);
+      options.addOption(OPTION_YARN_QUEUE_NAME, true, OPTION_YARN_QUEUE_NAME_DESC);
+
 
       HelpFormatter helpFormatter = new HelpFormatter();
       helpFormatter.setLeftPadding(2);


Mime
View raw message