hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From li...@apache.org
Subject svn commit: r1505597 - in /hbase/branches/0.89-fb/src: main/java/org/apache/hadoop/hbase/mapred/HexStringTableRecordReaderImpl.java test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatWithHFileOutputFormat.java
Date Mon, 22 Jul 2013 04:59:29 GMT
Author: liyin
Date: Mon Jul 22 04:59:29 2013
New Revision: 1505597

URL: http://svn.apache.org/r1505597
Log:
[master] Verifying that deletes work fine with HFileOutputFormat

Author: gauravm

Summary:
@groys wanted to know a good way to bulk delete. We found that HFileOutputFormat supports
Deletes as well, so making a slight modification in a unit test to
verify that.

Test Plan: Unit Test

Reviewers: liyintang, groys, manukranthk

Reviewed By: manukranthk

CC: hbase-eng@

Differential Revision: https://phabricator.fb.com/D890008

Task ID: 2614378

Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapred/HexStringTableRecordReaderImpl.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatWithHFileOutputFormat.java

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapred/HexStringTableRecordReaderImpl.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapred/HexStringTableRecordReaderImpl.java?rev=1505597&r1=1505596&r2=1505597&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapred/HexStringTableRecordReaderImpl.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapred/HexStringTableRecordReaderImpl.java
Mon Jul 22 04:59:29 2013
@@ -25,8 +25,8 @@ import java.math.BigInteger;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.hbase.util.Bytes;
 
-import com.sun.org.apache.commons.logging.Log;
-import com.sun.org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 public class HexStringTableRecordReaderImpl extends TableRecordReaderImpl {
   final Log LOG = LogFactory.getLog(HexStringTableRecordReaderImpl.class);

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatWithHFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatWithHFileOutputFormat.java?rev=1505597&r1=1505596&r2=1505597&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatWithHFileOutputFormat.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatWithHFileOutputFormat.java
Mon Jul 22 04:59:29 2013
@@ -27,9 +27,10 @@ public class TestTableInputFormatWithHFi
   static final String TABLENAME = "testMultipleHLogs";
   static final byte[] CF1 = Bytes.toBytes("cf1");
   static final byte[] CF2 = Bytes.toBytes("cf2");
-  static final byte[][] FAMILIES = new byte[][]{CF1, CF2};
+  static final byte[] CF3 = Bytes.toBytes("cf3");
+  static final byte[][] FAMILIES = new byte[][]{CF1, CF2, CF3};
   static final int REGION_NUM = 20;
-  static final byte[] QAULIFIER = Bytes.toBytes("q");
+  static final byte[] QUALIFIER = Bytes.toBytes("q");
   static final byte[] VALUE = Bytes.toBytes("v");
   static final Path OUTPUTPATH = new Path("TEST-OUTPUT");
   static HTable htable;
@@ -82,7 +83,10 @@ public class TestTableInputFormatWithHFi
     // Put some data for each Region
     for (byte[] row : htable.getStartKeys()) {
       Put p = new Put(row);
-      p.add(CF1, QAULIFIER, VALUE);
+      p.add(CF1, QUALIFIER, VALUE);
+      // We would delete the kv for CF3, to verify that Deletes work with
+      // HFileOutputFormat
+      p.add(CF3, QUALIFIER, VALUE);
       htable.put(p);
       htable.flushCommits();
     }
@@ -92,7 +96,8 @@ public class TestTableInputFormatWithHFi
     // Create the scan object
     Scan scan = new Scan();
     scan.addFamily(CF1);
-    
+    scan.addFamily(CF3);
+
     // Create and initialize the MR job
     Job job = new Job(conf, "process column contents");
     FileOutputFormat.setOutputPath(job, OUTPUTPATH);
@@ -120,20 +125,23 @@ public class TestTableInputFormatWithHFi
     Scan scan = new Scan();
     scan.addFamily(CF1);
     scan.addFamily(CF2);
+    scan.addFamily(CF3);
     
     ResultScanner s = htable.getScanner(scan);
     Result result = null;
     int count = 0;
     while((result = s.next()) != null) {
       count++;
-      
+
+      // We should only see CF1 and CF2. And not see CF3 since, we also added
+      // Delete kvs in the MR job.
       Assert.assertEquals(2, result.list().size());
       KeyValue kvFromCF1 = result.list().get(0);
       KeyValue kvFromCF2 = result.list().get(1);
-      
+
       Assert.assertTrue(Bytes.compareTo(kvFromCF1.getFamily(), CF1) == 0);
       Assert.assertTrue(Bytes.compareTo(kvFromCF2.getFamily(), CF2) == 0);
-      
+
       Assert.assertTrue(Bytes.compareTo(kvFromCF1.getRow(), kvFromCF2.getRow()) == 0);
       Assert.assertTrue(Bytes.compareTo(kvFromCF1.getQualifier(), kvFromCF2.getQualifier())
== 0);
       Assert.assertTrue(kvFromCF1.getTimestamp() == kvFromCF2.getTimestamp());
@@ -147,10 +155,13 @@ public class TestTableInputFormatWithHFi
     private KeyValue previousKV = null;
     public void map(ImmutableBytesWritable key, Result result, Context context)
     throws IOException, InterruptedException {
-      Assert.assertEquals(1, result.size());
+      Assert.assertEquals(2, result.size());
       KeyValue tmp = result.list().get(0);
       KeyValue currentKV = new KeyValue(tmp.getRow(), CF2, tmp.getQualifier(), tmp.getTimestamp(),
tmp.getValue());
-      
+
+      KeyValue tmp2 = result.list().get(1);
+      KeyValue deleteKV = new KeyValue(tmp2.getRow(), CF3, tmp2.getQualifier(), tmp.getTimestamp(),
KeyValue.Type.Delete);
+
       // Sanity check that the output key value is sorted
       if (previousKV != null) {
         Assert.assertTrue(KeyValue.COMPARATOR.compare(currentKV, previousKV) >= 0);
@@ -158,6 +169,7 @@ public class TestTableInputFormatWithHFi
       previousKV = currentKV;
       System.out.println("current KV: " + Bytes.toStringBinary(currentKV.getBuffer()));
       context.write(key, currentKV);
+      context.write(key, deleteKV);
     }
   }
 }



Mime
View raw message