Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@hbase.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: tedyu@apache.org
To: commits@hbase.apache.org
Message-Id: <274ec0cbf6444bf18c1a14b7229a0000@git.apache.org>
Subject: hbase git commit: HBASE-15862 Backup - Delete- Restore does not
 restore deleted data (Vladimir)
Date: Wed, 15 Jun 2016 04:35:14 +0000 (UTC)
archived-at: Wed, 15 Jun 2016 04:35:17 -0000

Repository: hbase
Updated Branches:
  refs/heads/HBASE-7912 235dfa321 -> 3f8cb39b1


HBASE-15862 Backup - Delete- Restore does not restore deleted data (Vladimir)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3f8cb39b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3f8cb39b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3f8cb39b

Branch: refs/heads/HBASE-7912
Commit: 3f8cb39b1f29714604d73862680d5d3028864d9d
Parents: 235dfa3
Author: tedyu <yuzhihong@gmail.com>
Authored: Tue Jun 14 21:34:58 2016 -0700
Committer: tedyu <yuzhihong@gmail.com>
Committed: Tue Jun 14 21:34:58 2016 -0700

----------------------------------------------------------------------
 .../hbase/backup/impl/RestoreClientImpl.java    | 15 +++--
 .../hbase/backup/util/RestoreServerUtil.java    | 35 ++++++----
 .../hbase/backup/TestBackupDeleteRestore.java   | 70 ++++++++++++++++++++
 3 files changed, 101 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/3f8cb39b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreClientImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreClientImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreClientImpl.java
index 9906f47..1f7305f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreClientImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreClientImpl.java
@@ -109,7 +109,7 @@ public final class RestoreClientImpl implements RestoreClient {
 
       // start restore process
       
-      restoreStage(backupManifestMap, sTableArray, tTableArray, autoRestore);
+      restoreStage(backupManifestMap, sTableArray, tTableArray, autoRestore, isOverwrite);
 
       LOG.info("Restore for " + Arrays.asList(sTableArray) + " are successful!");
 
@@ -209,8 +209,9 @@ public final class RestoreClientImpl implements RestoreClient {
    */
   private void restoreStage(
     HashMap<TableName, BackupManifest> backupManifestMap, TableName[] sTableArray,
-    TableName[] tTableArray, boolean autoRestore) throws IOException {
+    TableName[] tTableArray, boolean autoRestore, boolean isOverwrite) throws IOException {
     TreeSet<BackupImage> restoreImageSet = new TreeSet<BackupImage>();
+    boolean truncateIfExists = autoRestore && isOverwrite;
     try {
       for (int i = 0; i < sTableArray.length; i++) {
         TableName table = sTableArray[i];
@@ -224,7 +225,7 @@ public final class RestoreClientImpl implements RestoreClient {
           list.addAll(depList);
           TreeSet<BackupImage> restoreList = new TreeSet<BackupImage>(list);
           LOG.debug("need to clear merged Image. to be implemented in future jira");
-          restoreImages(restoreList.iterator(), table, tTableArray[i]);
+          restoreImages(restoreList.iterator(), table, tTableArray[i], truncateIfExists);
           restoreImageSet.addAll(restoreList);
         } else {
           BackupImage image = manifest.getBackupImage();
@@ -284,7 +285,7 @@ public final class RestoreClientImpl implements RestoreClient {
     if (manifest.getType() == BackupType.FULL || converted) {
       LOG.info("Restoring '" + sTable + "' to '" + tTable + "' from "
           + (converted ? "converted" : "full") + " backup image " + tableBackupPath.toString());
-      restoreTool.fullRestoreTable(tableBackupPath, sTable, tTable, converted);
+      restoreTool.fullRestoreTable(tableBackupPath, sTable, tTable, converted, false);
     } else { // incremental Backup
       String logBackupDir =
           HBackupFileSystem.getLogBackupDir(image.getRootDir(), image.getBackupId());
@@ -304,7 +305,8 @@ public final class RestoreClientImpl implements RestoreClient {
    * @param tTable: table to be restored to
    * @throws IOException exception
    */
-  private void restoreImages(Iterator<BackupImage> it, TableName sTable, TableName tTable)
+  private void restoreImages(Iterator<BackupImage> it, TableName sTable, 
+      TableName tTable, boolean truncateIfExists)
       throws IOException {
 
     // First image MUST be image of a FULL backup
@@ -326,7 +328,8 @@ public final class RestoreClientImpl implements RestoreClient {
     if (manifest.getType() == BackupType.FULL || converted) {
       LOG.info("Restoring '" + sTable + "' to '" + tTable + "' from "
           + (converted ? "converted" : "full") + " backup image " + tableBackupPath.toString());
-      restoreTool.fullRestoreTable(tableBackupPath, sTable, tTable, converted);
+      restoreTool.fullRestoreTable(tableBackupPath, sTable, tTable, 
+        converted, truncateIfExists);
       
     } else { // incremental Backup
       throw new IOException("Unexpected backup type " + image.getType());

http://git-wip-us.apache.org/repos/asf/hbase/blob/3f8cb39b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreServerUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreServerUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreServerUtil.java
index 37631bb..1b05aa9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreServerUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreServerUtil.java
@@ -167,8 +167,8 @@ public class RestoreServerUtil {
   }
 
   public void fullRestoreTable(Path tableBackupPath, TableName tableName, TableName newTableName,
-      boolean converted) throws IOException {
-    restoreTableAndCreate(tableName, newTableName, tableBackupPath, converted);
+      boolean converted, boolean truncateIfExists) throws IOException {
+    restoreTableAndCreate(tableName, newTableName, tableBackupPath, converted, truncateIfExists);
   }
 
   /**
@@ -275,7 +275,7 @@ public class RestoreServerUtil {
   }
 
   private void restoreTableAndCreate(TableName tableName, TableName newTableName,
-      Path tableBackupPath, boolean converted) throws IOException {
+      Path tableBackupPath, boolean converted, boolean truncateIfExists) throws IOException {
     if (newTableName == null || newTableName.equals("")) {
       newTableName = tableName;
     }
@@ -316,7 +316,8 @@ public class RestoreServerUtil {
             + ", will only create table");
         }
         tableDescriptor.setName(newTableName);
-        checkAndCreateTable(tableBackupPath, tableName, newTableName, null, tableDescriptor);
+        checkAndCreateTable(tableBackupPath, tableName, newTableName, null, 
+          tableDescriptor, truncateIfExists);
         return;
       } else {
         throw new IllegalStateException("Cannot restore hbase table because directory '"
@@ -339,7 +340,7 @@ public class RestoreServerUtil {
         // should only try to create the table with all region informations, so we could pre-split
         // the regions in fine grain
         checkAndCreateTable(tableBackupPath, tableName, newTableName, regionPathList,
-          tableDescriptor);
+          tableDescriptor, truncateIfExists);
         if (tableArchivePath != null) {
           // start real restore through bulkload
           // if the backup target is on local cluster, special action needed
@@ -570,28 +571,36 @@ public class RestoreServerUtil {
    * @throws IOException exception
    */
   private void checkAndCreateTable(Path tableBackupPath, TableName tableName,
-      TableName targetTableName, ArrayList<Path> regionDirList, HTableDescriptor htd)
+      TableName targetTableName, ArrayList<Path> regionDirList, 
+      HTableDescriptor htd, boolean truncateIfExists)
           throws IOException {
     HBaseAdmin hbadmin = null;
     Connection conn = null;
     try {
       conn = ConnectionFactory.createConnection(conf);
       hbadmin = (HBaseAdmin) conn.getAdmin();
+      boolean createNew = false;
       if (hbadmin.tableExists(targetTableName)) {
-        LOG.info("Using exising target table '" + targetTableName + "'");
+        if(truncateIfExists) {
+          LOG.info("Truncating exising target table '" + targetTableName +
+            "', preserving region splits");
+          hbadmin.disableTable(targetTableName);
+          hbadmin.truncateTable(targetTableName, true);
+        } else{
+          LOG.info("Using exising target table '" + targetTableName + "'");
+        }
       } else {
+        createNew = true;
+      }      
+      if(createNew){
         LOG.info("Creating target table '" + targetTableName + "'");
-
-        // if no region dir given, create the table and return
+        // if no region directory given, create the table and return
         if (regionDirList == null || regionDirList.size() == 0) {
-
           hbadmin.createTable(htd);
           return;
         }
-
         byte[][] keys = generateBoundaryKeys(regionDirList);
-
-        // create table using table decriptor and region boundaries
+        // create table using table descriptor and region boundaries
         hbadmin.createTable(htd, keys);
       }
     } catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/3f8cb39b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteRestore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteRestore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteRestore.java
new file mode 100644
index 0000000..702e8e7
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteRestore.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
+ * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
+ * for the specific language governing permissions and limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.BackupAdmin;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.google.common.collect.Lists;
+
+@Category(MediumTests.class)
+public class TestBackupDeleteRestore extends TestBackupBase {
+
+  private static final Log LOG = LogFactory.getLog(TestBackupDeleteRestore.class);
+
+  /**
+   * Verify that load data- backup - delete some data - restore
+   * works as expected - deleted data get restored.
+   * @throws Exception
+   */
+  @Test
+  public void testBackupDeleteRestore() throws Exception {
+
+    LOG.info("test full restore on a single table empty table");
+
+    List<TableName> tables = Lists.newArrayList(table1);
+    String backupId = fullTableBackup(tables);
+    assertTrue(checkSucceeded(backupId));
+    LOG.info("backup complete");
+    int numRows = TEST_UTIL.countRows(table1);
+    HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
+    // delete row    
+    try ( Table table = TEST_UTIL.getConnection().getTable(table1);) {      
+      Delete delete = new Delete("row0".getBytes());
+      table.delete(delete);
+      hba.flush(table1);
+    }   
+        
+    TableName[] tableset = new TableName[] { table1 };
+    TableName[] tablemap = null;//new TableName[] { table1_restore };
+    BackupAdmin client = getBackupAdmin();
+    client.restore(createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, true, tableset, tablemap, true));
+    
+    
+    int numRowsAfterRestore = TEST_UTIL.countRows(table1);
+    assertEquals( numRows, numRowsAfterRestore);    
+    hba.close();
+  }    
+}
\ No newline at end of file