hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r816035 - in /hadoop/hbase/branches/0.20: ./ src/java/org/apache/hadoop/hbase/mapreduce/
Date Thu, 17 Sep 2009 03:20:58 GMT
Author: stack
Date: Thu Sep 17 03:20:57 2009
New Revision: 816035

URL: http://svn.apache.org/viewvc?rev=816035&view=rev
Log:
HBASE-1684  Backup (Export/Import) contrib tool for 0.20

Modified:
    hadoop/hbase/branches/0.20/CHANGES.txt
    hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/Driver.java
    hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
    hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
    hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java

Modified: hadoop/hbase/branches/0.20/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/CHANGES.txt?rev=816035&r1=816034&r2=816035&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.20/CHANGES.txt Thu Sep 17 03:20:57 2009
@@ -30,6 +30,7 @@
    HBASE-1827  Add disabling block cache scanner flag to the shell
    HBASE-1574  Client and server APIs to do batch deletes
    HBASE-1833  hfile.main fixes
+   HBASE-1684  Backup (Export/Import) contrib tool for 0.20
 
 Release 0.20.0 - Tue Sep  8 12:48:41 PDT 2009
 

Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/Driver.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/Driver.java?rev=816035&r1=816034&r2=816035&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/Driver.java (original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/Driver.java Thu
Sep 17 03:20:57 2009
@@ -19,7 +19,6 @@
  */
 package org.apache.hadoop.hbase.mapreduce;
 
-import org.apache.hadoop.hbase.migration.nineteen.HStoreFileToStoreFile;
 import org.apache.hadoop.util.ProgramDriver;
 
 /**
@@ -35,9 +34,8 @@
     ProgramDriver pgd = new ProgramDriver();
     pgd.addClass(RowCounter.NAME, RowCounter.class,
       "Count rows in HBase table");
-    pgd.addClass(HStoreFileToStoreFile.JOBNAME,
-      HStoreFileToStoreFile.class,
-      "Bulk convert 0.19 HStoreFiles to 0.20 StoreFiles");
+    pgd.addClass(Export.NAME, Export.class, "Write table data to HDFS.");
+    pgd.addClass(Import.NAME, Import.class, "Import data written by Export.");
     pgd.driver(args);
   }
 }

Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java?rev=816035&r1=816034&r2=816035&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
(original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
Thu Sep 17 03:20:57 2009
@@ -22,14 +22,13 @@
 import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
 import org.apache.hadoop.util.GenericOptionsParser;
 
 /**
@@ -37,7 +36,6 @@
  * input row has columns that have content.
  */
 public class RowCounter {
-  
   /** Name of this 'program'. */
   static final String NAME = "rowcounter";
 
@@ -48,7 +46,7 @@
   extends TableMapper<ImmutableBytesWritable, Result> {
     
     /** Counter enumeration to count the actual rows. */
-    private static enum Counters { ROWS }
+    private static enum Counters {ROWS}
 
     /**
      * Maps the data.
@@ -84,11 +82,12 @@
    */
   public static Job createSubmittableJob(Configuration conf, String[] args) 
   throws IOException {
-    Job job = new Job(conf, NAME);
+    String tableName = args[0];
+    Job job = new Job(conf, NAME + "_" + tableName);
     job.setJarByClass(RowCounter.class);
     // Columns are space delimited
     StringBuilder sb = new StringBuilder();
-    final int columnoffset = 2;
+    final int columnoffset = 1;
     for (int i = columnoffset; i < args.length; i++) {
       if (i > columnoffset) {
         sb.append(" ");
@@ -96,13 +95,14 @@
       sb.append(args[i]);
     }
     Scan scan = new Scan();
-    scan.addColumns(sb.toString());
+    if (sb.length() > 0) scan.addColumns(sb.toString());
     // Second argument is the table name.
-    TableMapReduceUtil.initTableMapperJob(args[1], scan,
+    TableMapReduceUtil.initTableMapperJob(tableName, scan,
       RowCounterMapper.class, ImmutableBytesWritable.class, Result.class, job);
+    job.setOutputFormatClass(NullOutputFormat.class);
     job.setNumReduceTasks(0);
     // first argument is the output directory.
-    FileOutputFormat.setOutputPath(job, new Path(args[0]));
+    // FileOutputFormat.setOutputPath(job, new Path(args[0]));
     return job;
   }
 
@@ -115,13 +115,12 @@
   public static void main(String[] args) throws Exception {
     HBaseConfiguration conf = new HBaseConfiguration();
     String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
-    if (otherArgs.length < 3) {
+    if (otherArgs.length < 1) {
       System.err.println("ERROR: Wrong number of parameters: " + args.length);
-      System.err.println("Usage: " + NAME + 
-        " <outputdir> <tablename> <column1> [<column2>...]");
+      System.err.println("Usage: RowCounter <tablename> [<column1> <column2>...]");
       System.exit(-1);
     }
     Job job = createSubmittableJob(conf, otherArgs);
     System.exit(job.waitForCompletion(true) ? 0 : 1);
   }
-}
\ No newline at end of file
+}

Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java?rev=816035&r1=816034&r2=816035&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
(original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
Thu Sep 17 03:20:57 2009
@@ -21,17 +21,18 @@
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Writables;
 import org.apache.hadoop.mapreduce.InputFormat;
 import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.JobContext;
@@ -249,7 +250,7 @@
     if (trr == null) {
       trr = new TableRecordReader();
     }
-    Scan sc = new Scan(scan);
+    Scan sc = new Scan(this.scan);
     sc.setStartRow(tSplit.getStartRow());
     sc.setStopRow(tSplit.getEndRow());
     trr.setScan(sc);
@@ -277,9 +278,6 @@
     if (table == null) {
       throw new IOException("No table was provided.");
     }
-    if (!scan.hasFamilies()) {
-      throw new IOException("Expecting at least one column.");
-    }
     int realNumSplits = startKeys.length;
     InputSplit[] splits = new InputSplit[realNumSplits];
     int middle = startKeys.length / realNumSplits;
@@ -320,7 +318,7 @@
    * @return The internal scan instance.
    */
   public Scan getScan() {
-    if (scan == null) scan = new Scan();
+    if (this.scan == null) this.scan = new Scan();
     return scan;
   }
 

Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java?rev=816035&r1=816034&r2=816035&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
(original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
Thu Sep 17 03:20:57 2009
@@ -58,11 +58,11 @@
       Class<? extends WritableComparable> outputKeyClass, 
       Class<? extends Writable> outputValueClass, Job job) throws IOException {
     job.setInputFormatClass(TableInputFormat.class);
-    job.setMapOutputValueClass(outputValueClass);
-    job.setMapOutputKeyClass(outputKeyClass);
+    if (outputValueClass != null) job.setMapOutputValueClass(outputValueClass);
+    if (outputKeyClass != null) job.setMapOutputKeyClass(outputKeyClass);
     job.setMapperClass(mapper);
     job.getConfiguration().set(TableInputFormat.INPUT_TABLE, table);
-    job.getConfiguration().set(TableInputFormat.SCAN, 
+    job.getConfiguration().set(TableInputFormat.SCAN,
       convertScanToString(scan));
   }
 
@@ -125,7 +125,7 @@
     Class<? extends TableReducer> reducer, Job job, Class partitioner)
   throws IOException {
     job.setOutputFormatClass(TableOutputFormat.class);
-    job.setReducerClass(reducer);
+    if (reducer != null) job.setReducerClass(reducer);
     job.getConfiguration().set(TableOutputFormat.OUTPUT_TABLE, table);
     job.setOutputKeyClass(ImmutableBytesWritable.class);
     job.setOutputValueClass(Put.class);



Mime
View raw message