Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 5BF1B200D30 for ; Mon, 30 Oct 2017 18:36:05 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 5A57E160BF8; Mon, 30 Oct 2017 17:36:05 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 821581609D5 for ; Mon, 30 Oct 2017 18:36:04 +0100 (CET) Received: (qmail 62437 invoked by uid 500); 30 Oct 2017 17:36:03 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 62428 invoked by uid 99); 30 Oct 2017 17:36:03 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 30 Oct 2017 17:36:03 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 909F9DFBAD; Mon, 30 Oct 2017 17:36:03 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: tedyu@apache.org To: commits@hbase.apache.org Message-Id: <6984dd862ca04564841f60eb8eb79b38@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hbase git commit: HBASE-18870 Hbase Backup should set the details to MR job name (Vishal Khandelwal) Date: Mon, 30 Oct 2017 17:36:03 +0000 (UTC) archived-at: Mon, 30 Oct 2017 17:36:05 -0000 Repository: hbase Updated Branches: refs/heads/master a79b66b32 -> 6712f8f63 HBASE-18870 Hbase Backup should set the details to MR job name (Vishal Khandelwal) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6712f8f6 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6712f8f6 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6712f8f6 Branch: refs/heads/master Commit: 6712f8f632e6ece032ee45f679db9a42a69f805f Parents: a79b66b Author: tedyu Authored: Mon Oct 30 10:35:52 2017 -0700 Committer: tedyu Committed: Mon Oct 30 10:35:52 2017 -0700 ---------------------------------------------------------------------- .../hadoop/hbase/backup/BackupRestoreConstants.java | 1 + .../hadoop/hbase/backup/impl/FullTableBackupClient.java | 10 ++++++++++ .../hbase/backup/impl/IncrementalTableBackupClient.java | 12 ++++++++++++ .../hadoop/hbase/backup/impl/RestoreTablesClient.java | 5 +++++ .../apache/hadoop/hbase/snapshot/ExportSnapshot.java | 4 +++- 5 files changed, 31 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/6712f8f6/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java ---------------------------------------------------------------------- diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java index 4603019..e86766c 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java @@ -93,6 +93,7 @@ public interface BackupRestoreConstants { public static final String OPTION_YARN_QUEUE_NAME_DESC = "Yarn queue name to run backup create command on"; public static final String OPTION_YARN_QUEUE_NAME_RESTORE_DESC = "Yarn queue name to run backup restore command on"; + public final static String JOB_NAME_CONF_KEY = "mapreduce.job.name"; public static final String BACKUP_CONFIG_STRING = BackupRestoreConstants.BACKUP_ENABLE_KEY + "=true\n" + "hbase.master.logcleaner.plugins=" http://git-wip-us.apache.org/repos/asf/hbase/blob/6712f8f6/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java ---------------------------------------------------------------------- diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java index 481e30f..cb8e092 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java @@ -22,6 +22,7 @@ import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.BACKUP_ATTEM import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.BACKUP_MAX_ATTEMPTS_KEY; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.DEFAULT_BACKUP_ATTEMPTS_PAUSE_MS; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.DEFAULT_BACKUP_MAX_ATTEMPTS; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.JOB_NAME_CONF_KEY; import java.io.IOException; import java.util.HashMap; @@ -92,8 +93,15 @@ public class FullTableBackupClient extends TableBackupClient { args[2] = "-copy-to"; args[3] = backupInfo.getTableBackupDir(table); + String jobname = "Full-Backup_" + backupInfo.getBackupId() + "_" + table.getNameAsString(); + if (LOG.isDebugEnabled()) { + LOG.debug("Setting snapshot copy job name to : " + jobname); + } + conf.set(JOB_NAME_CONF_KEY, jobname); + LOG.debug("Copy snapshot " + args[1] + " to " + args[3]); res = copyService.copy(backupInfo, backupManager, conf, BackupType.FULL, args); + // if one snapshot export failed, do not continue for remained snapshots if (res != 0) { LOG.error("Exporting Snapshot " + args[1] + " failed with return code: " + res + "."); @@ -101,6 +109,8 @@ public class FullTableBackupClient extends TableBackupClient { throw new IOException("Failed of exporting snapshot " + args[1] + " to " + args[3] + " with reason code " + res); } + + conf.unset(JOB_NAME_CONF_KEY); LOG.info("Snapshot copy " + args[1] + " finished."); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/6712f8f6/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java ---------------------------------------------------------------------- diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java index 566b77d..00baeb7 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hbase.backup.impl; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.JOB_NAME_CONF_KEY; + import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -296,6 +298,13 @@ public class IncrementalTableBackupClient extends TableBackupClient { String[] strArr = new String[files.length + 1]; System.arraycopy(files, 0, strArr, 0, files.length); strArr[strArr.length - 1] = backupDest; + + String jobname = "Incremental_Backup-HFileCopy-" + backupInfo.getBackupId(); + if (LOG.isDebugEnabled()) { + LOG.debug("Setting incremental copy HFiles job name to : " + jobname); + } + conf.set(JOB_NAME_CONF_KEY, jobname); + BackupCopyJob copyService = BackupRestoreFactory.getBackupCopyJob(conf); int res = copyService.copy(backupInfo, backupManager, conf, BackupType.INCREMENTAL, strArr); if (res != 0) { @@ -353,10 +362,12 @@ public class IncrementalTableBackupClient extends TableBackupClient { // a Map task for each file. We use ';' as separator // because WAL file names contains ',' String dirs = StringUtils.join(dirPaths, ';'); + String jobname = "Incremental_Backup-" + backupId + "-" + tableName.getNameAsString(); Path bulkOutputPath = getBulkOutputDirForTable(tableName); conf.set(WALPlayer.BULK_OUTPUT_CONF_KEY, bulkOutputPath.toString()); conf.set(WALPlayer.INPUT_FILES_SEPARATOR_KEY, ";"); + conf.set(JOB_NAME_CONF_KEY, jobname); String[] playerArgs = { dirs, tableName.getNameAsString() }; try { @@ -366,6 +377,7 @@ public class IncrementalTableBackupClient extends TableBackupClient { throw new IOException("WAL Player failed"); } conf.unset(WALPlayer.INPUT_FILES_SEPARATOR_KEY); + conf.unset(JOB_NAME_CONF_KEY); } catch (IOException e) { throw e; } catch (Exception ee) { http://git-wip-us.apache.org/repos/asf/hbase/blob/6712f8f6/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreTablesClient.java ---------------------------------------------------------------------- diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreTablesClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreTablesClient.java index 1d32c88..fc0fdde 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreTablesClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreTablesClient.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hbase.backup.impl; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.JOB_NAME_CONF_KEY; + import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -150,8 +152,10 @@ public class RestoreTablesClient { if (manifest.getType() == BackupType.FULL) { LOG.info("Restoring '" + sTable + "' to '" + tTable + "' from full" + " backup image " + tableBackupPath.toString()); + conf.set(JOB_NAME_CONF_KEY, "Full_Restore-" + backupId + "-" + tTable); restoreTool.fullRestoreTable(conn, tableBackupPath, sTable, tTable, truncateIfExists, lastIncrBackupId); + conf.unset(JOB_NAME_CONF_KEY); } else { // incremental Backup throw new IOException("Unexpected backup type " + image.getType()); } @@ -175,6 +179,7 @@ public class RestoreTablesClient { LOG.info("Restoring '" + sTable + "' to '" + tTable + "' from log dirs: " + dirs); Path[] paths = new Path[dirList.size()]; dirList.toArray(paths); + conf.set(JOB_NAME_CONF_KEY, "Incremental_Restore-" + backupId + "-" + tTable); restoreTool.incrementalRestoreTable(conn, tableBackupPath, paths, new TableName[] { sTable }, new TableName[] { tTable }, lastIncrBackupId); LOG.info(sTable + " has been successfully restored to " + tTable); http://git-wip-us.apache.org/repos/asf/hbase/blob/6712f8f6/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java ---------------------------------------------------------------------- diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java index 5dd9bea..462d6bc 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java @@ -108,6 +108,7 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool { private static final String CONF_BUFFER_SIZE = "snapshot.export.buffer.size"; private static final String CONF_MAP_GROUP = "snapshot.export.default.map.group"; private static final String CONF_BANDWIDTH_MB = "snapshot.export.map.bandwidth.mb"; + private static final String CONF_MR_JOB_NAME = "mapreduce.job.name"; protected static final String CONF_SKIP_TMP = "snapshot.export.skip.tmp"; static class Testing { @@ -807,8 +808,9 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool { conf.set(CONF_SNAPSHOT_NAME, snapshotName); conf.set(CONF_SNAPSHOT_DIR, snapshotDir.toString()); + String jobname = conf.get(CONF_MR_JOB_NAME, "ExportSnapshot-" + snapshotName); Job job = new Job(conf); - job.setJobName("ExportSnapshot-" + snapshotName); + job.setJobName(jobname); job.setJarByClass(ExportSnapshot.class); TableMapReduceUtil.addDependencyJars(job); job.setMapperClass(ExportMapper.class);