hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From la...@apache.org
Subject svn commit: r1304110 - in /hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce: CopyTable.java Export.java Import.java ImportTsv.java RowCounter.java
Date Thu, 22 Mar 2012 22:34:36 GMT
Author: larsh
Date: Thu Mar 22 22:34:35 2012
New Revision: 1304110

URL: http://svn.apache.org/viewvc?rev=1304110&view=rev
Log:
HBASE-4657 Improve the efficiency of our MR jobs with a few configurations

Modified:
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java?rev=1304110&r1=1304109&r2=1304110&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java Thu Mar 22
22:34:35 2012
@@ -70,6 +70,7 @@ public class CopyTable {
     Job job = new Job(conf, NAME + "_" + tableName);
     job.setJarByClass(CopyTable.class);
     Scan scan = new Scan();
+    scan.setCacheBlocks(false);
     if (startTime != 0) {
       scan.setTimeRange(startTime,
           endTime == 0 ? HConstants.LATEST_TIMESTAMP : endTime);
@@ -115,7 +116,7 @@ public class CopyTable {
     if (errorMsg != null && errorMsg.length() > 0) {
       System.err.println("ERROR: " + errorMsg);
     }
-    System.err.println("Usage: CopyTable [--rs.class=CLASS] " +
+    System.err.println("Usage: CopyTable [general options] [--rs.class=CLASS] " +
         "[--rs.impl=IMPL] [--starttime=X] [--endtime=Y] " +
         "[--new.name=NEW] [--peer.adr=ADR] <tablename>");
     System.err.println();
@@ -144,6 +145,9 @@ public class CopyTable {
         "org.apache.hadoop.hbase.mapreduce.CopyTable --rs.class=org.apache.hadoop.hbase.ipc.ReplicationRegionInterface
" +
         "--rs.impl=org.apache.hadoop.hbase.regionserver.replication.ReplicationRegionServer
--starttime=1265875194289 --endtime=1265878794289 " +
         "--peer.adr=server1,server2,server3:2181:/hbase --families=myOldCf:myNewCf,cf2,cf3
TestTable ");
+    System.err.println("For performance consider the following general options:\n"
+        + "-Dhbase.client.scanner.caching=100\n"
+        + "-Dmapred.map.tasks.speculative.execution=false");
   }
 
   private static boolean doCommandLine(final String[] args) {

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java?rev=1304110&r1=1304109&r2=1304110&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java Thu Mar 22 22:34:35
2012
@@ -170,6 +170,10 @@ public class Export {
     System.err.println("  to control/limit what is exported..");
     System.err.println("   -D " + TableInputFormat.SCAN_COLUMN_FAMILY + "=<familyName>");
     System.err.println("   -D " + RAW_SCAN + "=true");
+    System.err.println("For performance consider the following properties:\n"
+        + "   -Dhbase.client.scanner.caching=100\n"
+        + "   -Dmapred.map.tasks.speculative.execution=false\n"
+        + "   -Dmapred.reduce.tasks.speculative.execution=false");
   }
 
   /**

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java?rev=1304110&r1=1304109&r2=1304110&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java Thu Mar 22 22:34:35
2012
@@ -275,6 +275,9 @@ public class Import {
     System.err.println("By default Import will load data directly into HBase. To instead
generate");
     System.err.println("HFiles of data to prepare for a bulk data load, pass the option:");
     System.err.println("  -D" + BULK_OUTPUT_CONF_KEY + "=/path/for/output");
+    System.err.println("For performance consider the following options:\n"
+        + "  -Dmapred.map.tasks.speculative.execution=false\n"
+        + "  -Dmapred.reduce.tasks.speculative.execution=false");
   }
 
   /**

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java?rev=1304110&r1=1304109&r2=1304110&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java Thu Mar 22
22:34:35 2012
@@ -270,7 +270,10 @@ public class ImportTsv {
       "  -D" + SKIP_LINES_CONF_KEY + "=false - fail if encountering an invalid line\n" +
       "  '-D" + SEPARATOR_CONF_KEY + "=|' - eg separate on pipes instead of tabs\n" +
       "  -D" + TIMESTAMP_CONF_KEY + "=currentTimeAsLong - use the specified timestamp for
the import\n" +
-      "  -D" + MAPPER_CONF_KEY + "=my.Mapper - A user-defined Mapper to use instead of "
+ DEFAULT_MAPPER.getName() + "\n";
+      "  -D" + MAPPER_CONF_KEY + "=my.Mapper - A user-defined Mapper to use instead of "
+ DEFAULT_MAPPER.getName() + "\n" +
+      "For performance consider the following options:\n" +
+      "  -Dmapred.map.tasks.speculative.execution=false\n" +
+      "  -Dmapred.reduce.tasks.speculative.execution=false";
 
     System.err.println(usage);
   }

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java?rev=1304110&r1=1304109&r2=1304110&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java Thu Mar 22
22:34:35 2012
@@ -113,6 +113,7 @@ public class RowCounter {
     Job job = new Job(conf, NAME + "_" + tableName);
     job.setJarByClass(RowCounter.class);
     Scan scan = new Scan();
+    scan.setCacheBlocks(false);
     if (startKey != null && !startKey.equals("")) {
       scan.setStartRow(Bytes.toBytes(startKey));
     }
@@ -149,8 +150,11 @@ public class RowCounter {
    * Prints usage without error message
    */
   private static void printUsage() {
-    System.err.println("Usage: RowCounter <tablename> " +
+    System.err.println("Usage: RowCounter [options] <tablename> " +
         "[--range=[startKey],[endKey]] [<column1> <column2>...]");
+    System.err.println("For performance consider the following options:\n"
+        + "-Dhbase.client.scanner.caching=100\n"
+        + "-Dmapred.map.tasks.speculative.execution=false");
   }
 
   /**



Mime
View raw message