hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From syuanji...@apache.org
Subject [12/50] [abbrv] hbase git commit: HBASE-8386 deprecate TableMapReduce.addDependencyJars(Configuration, class<?> ...)
Date Wed, 17 Aug 2016 18:34:18 GMT
HBASE-8386 deprecate TableMapReduce.addDependencyJars(Configuration, class<?> ...)

Signed-off-by: Matteo Bertozzi <matteo.bertozzi@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/431c8c9a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/431c8c9a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/431c8c9a

Branch: refs/heads/hbase-12439
Commit: 431c8c9ad03336b5ab844d97adc47b8a73d7aa00
Parents: c59f764
Author: Sean Busbey <busbey@apache.org>
Authored: Tue Jul 19 07:21:26 2016 -0500
Committer: Sean Busbey <busbey@apache.org>
Committed: Fri Aug 5 10:35:02 2016 -0500

----------------------------------------------------------------------
 .../test/IntegrationTestBigLinkedList.java      |  6 ++--
 .../test/IntegrationTestLoadAndVerify.java      |  4 +--
 ...tionTestWithCellVisibilityLoadAndVerify.java |  2 +-
 .../hadoop/hbase/mapred/TableMapReduceUtil.java |  2 +-
 .../apache/hadoop/hbase/mapreduce/Import.java   |  6 ++--
 .../hadoop/hbase/mapreduce/ImportTsv.java       |  2 +-
 .../hbase/mapreduce/TableMapReduceUtil.java     | 30 ++++++++++++++++++--
 .../hadoop/hbase/mapreduce/WALPlayer.java       |  4 +--
 .../hadoop/hbase/PerformanceEvaluation.java     |  2 +-
 .../mapred/TestTableSnapshotInputFormat.java    |  2 +-
 .../mapreduce/TestTableSnapshotInputFormat.java |  2 +-
 src/main/asciidoc/_chapters/mapreduce.adoc      |  2 +-
 12 files changed, 45 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
index 6c54dca..135bea7 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
@@ -785,7 +785,8 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase
{
 
       job.getConfiguration().setBoolean("mapreduce.map.speculative", false);
       TableMapReduceUtil.addDependencyJars(job);
-      TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
+      TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
+                                                     AbstractHBaseTool.class);
       TableMapReduceUtil.initCredentials(job);
 
       boolean success = jobCompletion(job);
@@ -1296,7 +1297,8 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase
{
 
       TableMapReduceUtil.initTableMapperJob(getTableName(getConf()).getName(), scan,
           VerifyMapper.class, BytesWritable.class, BytesWritable.class, job);
-      TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
+      TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
+                                                     AbstractHBaseTool.class);
 
       job.getConfiguration().setBoolean("mapreduce.map.speculative", false);
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java
index e279dfb..d12383d 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java
@@ -348,7 +348,7 @@ public void cleanUpCluster() throws Exception {
 
     TableMapReduceUtil.addDependencyJars(job);
 
-    TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
+    TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), AbstractHBaseTool.class);
     TableMapReduceUtil.initCredentials(job);
     assertTrue(job.waitForCompletion(true));
     return job;
@@ -372,7 +372,7 @@ public void cleanUpCluster() throws Exception {
     TableMapReduceUtil.initTableMapperJob(
         htd.getTableName().getNameAsString(), scan, VerifyMapper.class,
         BytesWritable.class, BytesWritable.class, job);
-    TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
+    TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), AbstractHBaseTool.class);
     int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING);
     TableMapReduceUtil.setScannerCaching(job, scannerCaching);
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java
index 52a705b..abaa481 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java
@@ -331,7 +331,7 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT
     scan.setAuthorizations(new Authorizations(auths));
     TableMapReduceUtil.initTableMapperJob(htd.getTableName().getNameAsString(), scan,
         VerifyMapper.class, NullWritable.class, NullWritable.class, job);
-    TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
+    TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), AbstractHBaseTool.class);
     int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING);
     TableMapReduceUtil.setScannerCaching(job, scannerCaching);
     job.setNumReduceTasks(0);

http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
index 84a279d..73340c4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
@@ -362,7 +362,7 @@ public class TableMapReduceUtil {
    */
   public static void addDependencyJars(JobConf job) throws IOException {
     org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addHBaseDependencyJars(job);
-    org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars(
+    org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJarsForClasses(
       job,
       // when making changes here, consider also mapreduce.TableMapReduceUtil
       // pull job classes

http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
index c7d5912..e2693b9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
@@ -616,7 +616,7 @@ public class Import extends Configured implements Tool {
     try {
       Class<? extends Filter> filter = conf.getClass(FILTER_CLASS_CONF_KEY, null, Filter.class);
       if (filter != null) {
-        TableMapReduceUtil.addDependencyJars(conf, filter);
+        TableMapReduceUtil.addDependencyJarsForClasses(conf, filter);
       }
     } catch (Exception e) {
       throw new IOException(e);
@@ -643,7 +643,7 @@ public class Import extends Configured implements Tool {
         fs.deleteOnExit(partitionsPath);
         job.setPartitionerClass(KeyValueWritableComparablePartitioner.class);
         job.setNumReduceTasks(regionLocator.getStartKeys().length);
-        TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
+        TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
             com.google.common.base.Preconditions.class);
       }
     } else if (hfileOutPath != null) {
@@ -658,7 +658,7 @@ public class Import extends Configured implements Tool {
         job.setMapOutputKeyClass(ImmutableBytesWritable.class);
         job.setMapOutputValueClass(KeyValue.class);
         HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator);
-        TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
+        TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
             com.google.common.base.Preconditions.class);
       }
     } else {

http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
index e778d99..39085df 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
@@ -587,7 +587,7 @@ public class ImportTsv extends Configured implements Tool {
               KeyValueSerialization.class.getName());
         }
         TableMapReduceUtil.addDependencyJars(job);
-        TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
+        TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
             com.google.common.base.Function.class /* Guava used by TsvParser */);
       }
     }

http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index 37e4e44..8d8a784 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -339,7 +339,7 @@ public class TableMapReduceUtil {
 
     if (addDependencyJars) {
       addDependencyJars(job);
-      addDependencyJars(job.getConfiguration(), MetricRegistry.class);
+      addDependencyJarsForClasses(job.getConfiguration(), MetricRegistry.class);
     }
 
     resetCacheConfig(job.getConfiguration());
@@ -787,7 +787,7 @@ public class TableMapReduceUtil {
           "  Continuing without it.");
     }
 
-    addDependencyJars(conf,
+    addDependencyJarsForClasses(conf,
       // explicitly pull a class from each module
       org.apache.hadoop.hbase.HConstants.class,                      // hbase-common
       org.apache.hadoop.hbase.protobuf.generated.ClientProtos.class, // hbase-protocol
@@ -835,7 +835,7 @@ public class TableMapReduceUtil {
   public static void addDependencyJars(Job job) throws IOException {
     addHBaseDependencyJars(job.getConfiguration());
     try {
-      addDependencyJars(job.getConfiguration(),
+      addDependencyJarsForClasses(job.getConfiguration(),
           // when making changes here, consider also mapred.TableMapReduceUtil
           // pull job classes
           job.getMapOutputKeyClass(),
@@ -855,9 +855,33 @@ public class TableMapReduceUtil {
    * Add the jars containing the given classes to the job's configuration
    * such that JobClient will ship them to the cluster and add them to
    * the DistributedCache.
+   * @deprecated rely on {@link #addDependencyJars(Job)} instead.
    */
+  @Deprecated
   public static void addDependencyJars(Configuration conf,
       Class<?>... classes) throws IOException {
+    LOG.warn("The addDependencyJars(Configuration, Class<?>...) method has been deprecated
since it"
+             + " is easy to use incorrectly. Most users should rely on addDependencyJars(Job)
" +
+             "instead. See HBASE-8386 for more details.");
+    addDependencyJarsForClasses(conf, classes);
+  }
+
+  /**
+   * Add the jars containing the given classes to the job's configuration
+   * such that JobClient will ship them to the cluster and add them to
+   * the DistributedCache.
+   *
+   * N.B. that this method at most adds one jar per class given. If there is more than one
+   * jar available containing a class with the same name as a given class, we don't define
+   * which of those jars might be chosen.
+   *
+   * @param conf The Hadoop Configuration to modify
+   * @param classes will add just those dependencies needed to find the given classes
+   * @throws IOException if an underlying library call fails.
+   */
+  @InterfaceAudience.Private
+  public static void addDependencyJarsForClasses(Configuration conf,
+      Class<?>... classes) throws IOException {
 
     FileSystem localFs = FileSystem.getLocal(conf);
     Set<String> jars = new HashSet<String>();

http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
index c6fefb2..452714b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
@@ -282,7 +282,7 @@ public class WALPlayer extends Configured implements Tool {
           RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
         HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator);
       }
-      TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
+      TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
           com.google.common.base.Preconditions.class);
     } else {
       // output to live cluster
@@ -295,7 +295,7 @@ public class WALPlayer extends Configured implements Tool {
     }
     String codecCls = WALCellCodec.getWALCellCodecClass(conf);
     try {
-      TableMapReduceUtil.addDependencyJars(job.getConfiguration(), Class.forName(codecCls));
+      TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), Class.forName(codecCls));
     } catch (Exception e) {
       throw new IOException("Cannot determine wal codec class " + codecCls, e);
     }

http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index 0528975..dc8a783 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -499,7 +499,7 @@ public class PerformanceEvaluation extends Configured implements Tool
{
     TextOutputFormat.setOutputPath(job, new Path(inputDir.getParent(), "outputs"));
 
     TableMapReduceUtil.addDependencyJars(job);
-    TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
+    TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
       Histogram.class,     // yammer metrics
       ObjectMapper.class); // jackson-mapper-asl
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java
index c85cead..c4fdea4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java
@@ -243,7 +243,7 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa
       JobConf jobConf = new JobConf(util.getConfiguration());
 
       jobConf.setJarByClass(util.getClass());
-      org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars(jobConf,
+      org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJarsForClasses(jobConf,
         TestTableSnapshotInputFormat.class);
 
       TableMapReduceUtil.initTableSnapshotMapJob(snapshotName, COLUMNS,

http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
index 7c10def..5e8c8cd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
@@ -281,7 +281,7 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa
       Scan scan = new Scan(startRow, endRow); // limit the scan
 
       job.setJarByClass(util.getClass());
-      TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
+      TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
         TestTableSnapshotInputFormat.class);
 
       TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName,

http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/src/main/asciidoc/_chapters/mapreduce.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/mapreduce.adoc b/src/main/asciidoc/_chapters/mapreduce.adoc
index 75718fd..dfa843a 100644
--- a/src/main/asciidoc/_chapters/mapreduce.adoc
+++ b/src/main/asciidoc/_chapters/mapreduce.adoc
@@ -73,7 +73,7 @@ This example assumes you use a BASH-compatible shell.
 $ HADOOP_CLASSPATH=`${HBASE_HOME}/bin/hbase classpath` ${HADOOP_HOME}/bin/hadoop jar ${HBASE_HOME}/lib/hbase-server-VERSION.jar
rowcounter usertable
 ----
 
-When the command runs, internally, the HBase JAR finds the dependencies it needs for ZooKeeper,
Guava, and its other dependencies on the passed `HADOOP_CLASSPATH` and adds the JARs to the
MapReduce job configuration.
+When the command runs, internally, the HBase JAR finds the dependencies it needs and adds
them to the MapReduce job configuration.
 See the source at `TableMapReduceUtil#addDependencyJars(org.apache.hadoop.mapreduce.Job)`
for how this is done.
 
 The command `hbase mapredcp` can also help you dump the CLASSPATH entries required by MapReduce,
which are the same jars `TableMapReduceUtil#addDependencyJars` would add.


Mime
View raw message