hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject git commit: HBASE-3135 Make our MR jobs implement Tool and use ToolRunner so can do -D trickery, etc. (Talat Uyarer)
Date Mon, 28 Jul 2014 23:39:56 GMT
Repository: hbase
Updated Branches:
  refs/heads/master c71d1f6a0 -> ff655e04d


HBASE-3135 Make our MR jobs implement Tool and use ToolRunner so can do -D trickery, etc.
(Talat Uyarer)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ff655e04
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ff655e04
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ff655e04

Branch: refs/heads/master
Commit: ff655e04d17051012a1645a82df42b5aec94af92
Parents: c71d1f6
Author: stack <stack@apache.org>
Authored: Mon Jul 28 16:39:49 2014 -0700
Committer: stack <stack@apache.org>
Committed: Mon Jul 28 16:39:49 2014 -0700

----------------------------------------------------------------------
 .../hadoop/hbase/mapreduce/CellCounter.java     | 36 +++++++++++-------
 .../hadoop/hbase/mapreduce/CopyTable.java       | 12 +++---
 .../apache/hadoop/hbase/mapreduce/Export.java   | 33 +++++++++-------
 .../apache/hadoop/hbase/mapreduce/Import.java   | 40 ++++++++++++--------
 .../hadoop/hbase/mapreduce/ImportTsv.java       |  2 +-
 .../hadoop/hbase/mapreduce/RowCounter.java      | 38 +++++++++++--------
 .../hadoop/hbase/mapreduce/WALPlayer.java       |  2 +-
 7 files changed, 97 insertions(+), 66 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/ff655e04/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java
index 2661bed..f4916ae 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java
@@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
@@ -45,6 +46,8 @@ import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
 import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 
 import com.google.common.base.Preconditions;
 
@@ -68,7 +71,7 @@ import com.google.common.base.Preconditions;
  */
 @InterfaceAudience.Public
 @InterfaceStability.Stable
-public class CellCounter {
+public class CellCounter extends Configured implements Tool {
   private static final Log LOG =
     LogFactory.getLog(CellCounter.class.getName());
 
@@ -185,7 +188,7 @@ public class CellCounter {
     Path outputDir = new Path(args[1]);
     String reportSeparatorString = (args.length > 2) ? args[2]: ":";
     conf.set("ReportSeparator", reportSeparatorString);
-    Job job = new Job(conf, NAME + "_" + tableName);
+    Job job = Job.getInstance(conf, NAME + "_" + tableName);
     job.setJarByClass(CellCounter.class);
     Scan scan = getConfiguredScanForJob(conf, args);
     TableMapReduceUtil.initTableMapperJob(tableName, scan,
@@ -233,15 +236,9 @@ public class CellCounter {
     return rowFilter;
   }
 
-  /**
-   * Main entry point.
-   *
-   * @param args The command line parameters.
-   * @throws Exception When running the job fails.
-   */
-  public static void main(String[] args) throws Exception {
-    Configuration conf = HBaseConfiguration.create();
-    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+  @Override
+  public int run(String[] args) throws Exception {
+    String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs();
     if (otherArgs.length < 1) {
       System.err.println("ERROR: Wrong number of parameters: " + args.length);
       System.err.println("Usage: CellCounter <tablename> <outputDir> <reportSeparator>
" +
@@ -254,9 +251,20 @@ public class CellCounter {
           "string : used to separate the rowId/column family name and qualifier name.");
       System.err.println(" [^[regex pattern] or [Prefix] parameter can be used to limit the
cell counter count " +
           "operation to a limited subset of rows from the table based on regex or prefix
pattern.");
-      System.exit(-1);
+      return -1;
     }
-    Job job = createSubmittableJob(conf, otherArgs);
-    System.exit(job.waitForCompletion(true) ? 0 : 1);
+    Job job = createSubmittableJob(getConf(), otherArgs);
+    return (job.waitForCompletion(true) ? 0 : 1);
+  }
+
+  /**
+   * Main entry point.
+   * @param args The command line parameters.
+   * @throws Exception When running the job fails.
+   */
+  public static void main(String[] args) throws Exception {
+    int errCode = ToolRunner.run(HBaseConfiguration.create(), new CellCounter(), args);
+    System.exit(errCode);
   }
+
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/ff655e04/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
index 984f3b5..decb6a7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
@@ -18,23 +18,23 @@
  */
 package org.apache.hadoop.hbase.mapreduce;
 
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
 /**
  * Tool used to copy a table to another one which can be on a different setup.
  * It is also configurable with a start and time as well as a specification
@@ -72,7 +72,7 @@ public class CopyTable extends Configured implements Tool {
     if (!doCommandLine(args)) {
       return null;
     }
-    Job job = new Job(conf, NAME + "_" + tableName);
+    Job job = Job.getInstance(conf, NAME + "_" + tableName);
     job.setJarByClass(CopyTable.class);
     Scan scan = new Scan();
     scan.setCacheBlocks(false);

http://git-wip-us.apache.org/repos/asf/hbase/blob/ff655e04/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
index a69d4e4..fdff068 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
@@ -25,7 +25,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.client.Result;
@@ -42,6 +42,8 @@ import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
 import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 
 /**
 * Export an HBase table.
@@ -50,7 +52,7 @@ import org.apache.hadoop.util.GenericOptionsParser;
 */
 @InterfaceAudience.Public
 @InterfaceStability.Stable
-public class Export {
+public class Export extends Configured implements Tool {
   private static final Log LOG = LogFactory.getLog(Export.class);
   final static String NAME = "export";
   final static String RAW_SCAN = "hbase.mapreduce.include.deleted.rows";
@@ -68,7 +70,7 @@ public class Export {
   throws IOException {
     String tableName = args[0];
     Path outputDir = new Path(args[1]);
-    Job job = new Job(conf, NAME + "_" + tableName);
+    Job job = Job.getInstance(conf, NAME + "_" + tableName);
     job.setJobName(NAME + "_" + tableName);
     job.setJarByClass(Export.class);
     // Set optional scan parameters
@@ -172,20 +174,25 @@ public class Export {
         + "   -D" + EXPORT_BATCHING + "=10");
   }
 
+
+  @Override
+  public int run(String[] args) throws Exception {
+    String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs();
+    if (otherArgs.length < 2) {
+      usage("Wrong number of arguments: " + otherArgs.length);
+      return -1;
+    }
+    Job job = createSubmittableJob(getConf(), otherArgs);
+    return (job.waitForCompletion(true) ? 0 : 1);
+  }
+
   /**
    * Main entry point.
-   *
-   * @param args  The command line parameters.
+   * @param args The command line parameters.
    * @throws Exception When running the job fails.
    */
   public static void main(String[] args) throws Exception {
-    Configuration conf = HBaseConfiguration.create();
-    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
-    if (otherArgs.length < 2) {
-      usage("Wrong number of arguments: " + otherArgs.length);
-      System.exit(-1);
-    }
-    Job job = createSubmittableJob(conf, otherArgs);
-    System.exit(job.waitForCompletion(true)? 0 : 1);
+    int errCode = ToolRunner.run(HBaseConfiguration.create(), new Export(), args);
+    System.exit(errCode);
   }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/ff655e04/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
index 6e783e9..7751c11 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
@@ -33,6 +33,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
@@ -57,6 +58,8 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 import org.apache.zookeeper.KeeperException;
 
 
@@ -65,7 +68,7 @@ import org.apache.zookeeper.KeeperException;
  */
 @InterfaceAudience.Public
 @InterfaceStability.Stable
-public class Import {
+public class Import extends Configured implements Tool {
   private static final Log LOG = LogFactory.getLog(Import.class);
   final static String NAME = "import";
   public final static String CF_RENAME_PROP = "HBASE_IMPORTER_RENAME_CFS";
@@ -399,7 +402,7 @@ public class Import {
     String tableName = args[0];
     conf.set(TABLE_NAME, tableName);
     Path inputDir = new Path(args[1]);
-    Job job = new Job(conf, NAME + "_" + tableName);
+    Job job = Job.getInstance(conf, NAME + "_" + tableName);
     job.setJarByClass(Importer.class);
     FileInputFormat.setInputPaths(job, inputDir);
     job.setInputFormatClass(SequenceFileInputFormat.class);
@@ -491,29 +494,34 @@ public class Import {
     }
   }
 
-  /**
-   * Main entry point.
-   *
-   * @param args  The command line parameters.
-   * @throws Exception When running the job fails.
-   */
-  public static void main(String[] args) throws Exception {
-    Configuration conf = HBaseConfiguration.create();
-    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+  @Override
+  public int run(String[] args) throws Exception {
+    String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs();
     if (otherArgs.length < 2) {
       usage("Wrong number of arguments: " + otherArgs.length);
-      System.exit(-1);
+      return -1;
     }
     String inputVersionString = System.getProperty(ResultSerialization.IMPORT_FORMAT_VER);
     if (inputVersionString != null) {
-      conf.set(ResultSerialization.IMPORT_FORMAT_VER, inputVersionString);
+      getConf().set(ResultSerialization.IMPORT_FORMAT_VER, inputVersionString);
     }
-    Job job = createSubmittableJob(conf, otherArgs);
+    Job job = createSubmittableJob(getConf(), otherArgs);
     boolean isJobSuccessful = job.waitForCompletion(true);
     if(isJobSuccessful){
       // Flush all the regions of the table
-      flushRegionsIfNecessary(conf);
+      flushRegionsIfNecessary(getConf());
     }
-    System.exit(job.waitForCompletion(true) ? 0 : 1);
+    return (isJobSuccessful ? 0 : 1);
+  }
+
+  /**
+   * Main entry point.
+   * @param args The command line parameters.
+   * @throws Exception When running the job fails.
+   */
+  public static void main(String[] args) throws Exception {
+    int errCode = ToolRunner.run(HBaseConfiguration.create(), new Import(), args);
+    System.exit(errCode);
   }
+
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/ff655e04/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
index 7bbf71f..0748481 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
@@ -416,7 +416,7 @@ public class ImportTsv extends Configured implements Tool {
     String tableName = args[0];
     Path inputDir = new Path(args[1]);
     String jobName = conf.get(JOB_NAME_CONF_KEY,NAME + "_" + tableName);
-    Job job = new Job(conf, jobName);
+    Job job = Job.getInstance(conf, NAME + "_" + tableName);
     job.setJarByClass(mapperClass);
     FileInputFormat.setInputPaths(job, inputDir);
     job.setInputFormatClass(TextInputFormat.class);

http://git-wip-us.apache.org/repos/asf/hbase/blob/ff655e04/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
index 0851415..c0cf32c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
@@ -26,6 +26,7 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
@@ -36,6 +37,8 @@ import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
 import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 
 /**
  * A job with a just a map phase to count rows. Map outputs table rows IF the
@@ -43,7 +46,7 @@ import org.apache.hadoop.util.GenericOptionsParser;
  */
 @InterfaceAudience.Public
 @InterfaceStability.Stable
-public class RowCounter {
+public class RowCounter extends Configured implements Tool {
 
   /** Name of this 'program'. */
   static final String NAME = "rowcounter";
@@ -112,7 +115,7 @@ public class RowCounter {
       }
     }
 
-    Job job = new Job(conf, NAME + "_" + tableName);
+    Job job = Job.getInstance(conf, NAME + "_" + tableName);
     job.setJarByClass(RowCounter.class);
     Scan scan = new Scan();
     scan.setCacheBlocks(false);
@@ -170,23 +173,28 @@ public class RowCounter {
         + "-Dmapreduce.map.speculative=false");
   }
 
-  /**
-   * Main entry point.
-   *
-   * @param args  The command line parameters.
-   * @throws Exception When running the job fails.
-   */
-  public static void main(String[] args) throws Exception {
-    Configuration conf = HBaseConfiguration.create();
-    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+  @Override
+  public int run(String[] args) throws Exception {
+    String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs();
     if (otherArgs.length < 1) {
       printUsage("Wrong number of parameters: " + args.length);
-      System.exit(-1);
+      return -1;
     }
-    Job job = createSubmittableJob(conf, otherArgs);
+    Job job = createSubmittableJob(getConf(), otherArgs);
     if (job == null) {
-      System.exit(-1);
+      return -1;
     }
-    System.exit(job.waitForCompletion(true) ? 0 : 1);
+    return (job.waitForCompletion(true) ? 0 : 1);
   }
+
+  /**
+   * Main entry point.
+   * @param args The command line parameters.
+   * @throws Exception When running the job fails.
+   */
+  public static void main(String[] args) throws Exception {
+    int errCode = ToolRunner.run(HBaseConfiguration.create(), new RowCounter(), args);
+    System.exit(errCode);
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/ff655e04/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
index 456d04f..3f0cde4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
@@ -231,7 +231,7 @@ public class WALPlayer extends Configured implements Tool {
     }
     conf.setStrings(TABLES_KEY, tables);
     conf.setStrings(TABLE_MAP_KEY, tableMap);
-    Job job = new Job(conf, NAME + "_" + inputDir);
+    Job job = Job.getInstance(conf, NAME + "_" + inputDir);
     job.setJarByClass(WALPlayer.class);
     FileInputFormat.setInputPaths(job, inputDir);
     job.setInputFormatClass(HLogInputFormat.class);


Mime
View raw message