hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject git commit: HBASE-12065 Import tool is not restoring multiple DeleteFamily markers of a row (Maddineni Sukumar)
Date Wed, 01 Oct 2014 23:40:07 GMT
Repository: hbase
Updated Branches:
  refs/heads/0.94 68ed793a1 -> 14eb97c8d


HBASE-12065 Import tool is not restoring multiple DeleteFamily markers of a row (Maddineni
Sukumar)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/14eb97c8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/14eb97c8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/14eb97c8

Branch: refs/heads/0.94
Commit: 14eb97c8d92a6e20119975cf8d240bb321c6ee03
Parents: 68ed793
Author: Andrew Purtell <apurtell@apache.org>
Authored: Wed Oct 1 14:53:03 2014 -0700
Committer: Andrew Purtell <apurtell@apache.org>
Committed: Wed Oct 1 16:37:27 2014 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hbase/mapreduce/Import.java   | 14 ++-
 .../hbase/mapreduce/TestImportExport.java       | 97 +++++++++++++++++++-
 2 files changed, 107 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/14eb97c8/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
index 433eaca..d1ceaff 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
@@ -148,7 +148,19 @@ public class Import {
   
           kv = convertKv(kv, cfRenameMap);
           // Deletes and Puts are gathered and written when finished
-          if (kv.isDelete()) {
+          /*
+           * If there are sequence of mutations and tombstones in an Export, and after Import
the same
+           * sequence should be restored as it is. If we combine all Delete tombstones into
single
+           * request then there is chance of ignoring few DeleteFamily tombstones, because
if we
+           * submit multiple DeleteFamily tombstones in single Delete request then we are
maintaining
+           * only newest in hbase table and ignoring other. Check - HBASE-12065
+           */
+          if (kv.isDeleteFamily()) {
+            Delete deleteFamily = new Delete(key.get());
+            deleteFamily.addDeleteMarker(kv);
+            deleteFamily.setClusterId(clusterId);
+            context.write(key, deleteFamily);
+          } else if (kv.isDelete()) {
             if (delete == null) {
               delete = new Delete(key.get());
             }

http://git-wip-us.apache.org/repos/asf/hbase/blob/14eb97c8/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
index 569364c..d77b9a4 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
@@ -17,9 +17,7 @@
  */
 package org.apache.hadoop.hbase.mapreduce;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
 
 import java.io.IOException;
 
@@ -321,6 +319,99 @@ public class TestImportExport {
     exportTable.close();
     importTable.close();
   }
+  
+  
+  @Test
+  public void testWithMultipleDeleteFamilyMarkersOfSameRowSameFamily() throws Exception {
+    String EXPORT_TABLE = "exportWithMultipleDeleteFamilyMarkersOfSameRowSameFamily";
+    HTableDescriptor desc = new HTableDescriptor(EXPORT_TABLE);
+    desc.addFamily(new HColumnDescriptor(FAMILYA)
+        .setMaxVersions(5)
+        .setKeepDeletedCells(true)
+    );
+    UTIL.getHBaseAdmin().createTable(desc);
+    HTable exportT = new HTable(UTIL.getConfiguration(), EXPORT_TABLE);
+
+    //Add first version of QUAL
+    Put p = new Put(ROW1);
+    p.add(FAMILYA, QUAL, now, QUAL);
+    exportT.put(p);
+
+    //Add Delete family marker
+    Delete d = new Delete(ROW1, now+3);
+    exportT.delete(d);
+
+    //Add second version of QUAL
+    p = new Put(ROW1);
+    p.add(FAMILYA, QUAL, now+5, "s".getBytes());
+    exportT.put(p);
+
+    //Add second Delete family marker
+    d = new Delete(ROW1, now+7);
+    exportT.delete(d);
+
+    String[] args = new String[] {
+        "-D" + Export.RAW_SCAN + "=true",
+        EXPORT_TABLE,
+        OUTPUT_DIR,
+        "1000", // max number of key versions per key to export
+    };
+
+    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
+        UTIL.getConfiguration()), args);
+    Configuration conf = opts.getConfiguration();
+    args = opts.getRemainingArgs();
+
+    Job job = Export.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    job.waitForCompletion(false);
+    assertTrue(job.isSuccessful());
+
+    String IMPORT_TABLE = "importWithMultipleDeleteFamilyMarkersOfSameRowSameFamily";
+    desc = new HTableDescriptor(IMPORT_TABLE);
+    desc.addFamily(new HColumnDescriptor(FAMILYA)
+        .setMaxVersions(5)
+        .setKeepDeletedCells(true)
+    );
+    UTIL.getHBaseAdmin().createTable(desc);
+    
+    HTable importT = new HTable(UTIL.getConfiguration(), IMPORT_TABLE);
+    args = new String[] {
+        IMPORT_TABLE,
+        OUTPUT_DIR
+    };
+
+    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+    conf = opts.getConfiguration();
+    args = opts.getRemainingArgs();
+
+    job = Import.createSubmittableJob(conf, args);
+    job.getConfiguration().set("mapreduce.framework.name", "yarn");
+    job.waitForCompletion(false);
+    assertTrue(job.isSuccessful());
+
+    Scan s = new Scan();
+    s.setMaxVersions();
+    s.setRaw(true);
+    
+    ResultScanner importedTScanner = importT.getScanner(s);
+    Result importedTResult = importedTScanner.next();
+    
+    ResultScanner exportedTScanner = exportT.getScanner(s);
+    Result  exportedTResult =  exportedTScanner.next();
+    try
+    {
+      Result.compareResults(exportedTResult, importedTResult);
+    }
+    catch (Exception e) {
+      fail("Original and imported tables data comparision failed with error:"+e.getMessage());
+    }
+    finally
+    {
+      exportT.close();
+      importT.close();
+    }
+  }
 
   /**
    * Count the number of keyvalues in the specified table for the given timerange


Mime
View raw message