hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [19/36] hbase git commit: HBASE-17614: Move Backup/Restore into separate module (Vladimir Rodionov)
Date Wed, 23 Aug 2017 16:47:16 GMT
http://git-wip-us.apache.org/repos/asf/hbase/blob/37c65946/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithFailures.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithFailures.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithFailures.java
deleted file mode 100644
index 7011ed3..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithFailures.java
+++ /dev/null
@@ -1,336 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import static org.apache.hadoop.hbase.backup.util.BackupUtils.succeeded;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl;
-import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
-import org.apache.hadoop.hbase.backup.mapreduce.MapReduceBackupMergeJob;
-import org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob;
-import org.apache.hadoop.hbase.backup.util.BackupUtils;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.util.Pair;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-import com.google.common.collect.Lists;
-
-@Category(LargeTests.class)
-public class TestIncrementalBackupMergeWithFailures extends TestBackupBase {
-  private static final Log LOG = LogFactory.getLog(TestIncrementalBackupMergeWithFailures.class);
-
-  static enum FailurePhase {
-    PHASE1, PHASE2, PHASE3, PHASE4
-  }
-  public final static String FAILURE_PHASE_KEY = "failurePhase";
-
-  static class BackupMergeJobWithFailures extends MapReduceBackupMergeJob {
-
-    FailurePhase failurePhase;
-
-    @Override
-    public void setConf(Configuration conf) {
-      super.setConf(conf);
-      String val = conf.get(FAILURE_PHASE_KEY);
-      if (val != null) {
-        failurePhase = FailurePhase.valueOf(val);
-      } else {
-        Assert.fail("Failure phase is not set");
-      }
-    }
-
-
-    /**
-     * This is the exact copy of parent's run() with injections
-     * of different types of failures
-     */
-    @Override
-    public void run(String[] backupIds) throws IOException {
-      String bulkOutputConfKey;
-
-      // TODO : run player on remote cluster
-      player = new MapReduceHFileSplitterJob();
-      bulkOutputConfKey = MapReduceHFileSplitterJob.BULK_OUTPUT_CONF_KEY;
-      // Player reads all files in arbitrary directory structure and creates
-      // a Map task for each file
-      String bids = StringUtils.join(backupIds, ",");
-
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Merge backup images " + bids);
-      }
-
-      List<Pair<TableName, Path>> processedTableList = new ArrayList<Pair<TableName, Path>>();
-      boolean finishedTables = false;
-      Connection conn = ConnectionFactory.createConnection(getConf());
-      BackupSystemTable table = new BackupSystemTable(conn);
-      FileSystem fs = FileSystem.get(getConf());
-
-      try {
-
-        // Start backup exclusive operation
-        table.startBackupExclusiveOperation();
-        // Start merge operation
-        table.startMergeOperation(backupIds);
-
-        // Select most recent backup id
-        String mergedBackupId = findMostRecentBackupId(backupIds);
-
-        TableName[] tableNames = getTableNamesInBackupImages(backupIds);
-        String backupRoot = null;
-
-        BackupInfo bInfo = table.readBackupInfo(backupIds[0]);
-        backupRoot = bInfo.getBackupRootDir();
-        // PHASE 1
-        checkFailure(FailurePhase.PHASE1);
-
-        for (int i = 0; i < tableNames.length; i++) {
-
-          LOG.info("Merge backup images for " + tableNames[i]);
-
-          // Find input directories for table
-
-          Path[] dirPaths = findInputDirectories(fs, backupRoot, tableNames[i], backupIds);
-          String dirs = StringUtils.join(dirPaths, ",");
-          Path bulkOutputPath =
-              BackupUtils.getBulkOutputDir(BackupUtils.getFileNameCompatibleString(tableNames[i]),
-                getConf(), false);
-          // Delete content if exists
-          if (fs.exists(bulkOutputPath)) {
-            if (!fs.delete(bulkOutputPath, true)) {
-              LOG.warn("Can not delete: " + bulkOutputPath);
-            }
-          }
-          Configuration conf = getConf();
-          conf.set(bulkOutputConfKey, bulkOutputPath.toString());
-          String[] playerArgs = { dirs, tableNames[i].getNameAsString() };
-
-          int result = 0;
-          // PHASE 2
-          checkFailure(FailurePhase.PHASE2);
-          player.setConf(getConf());
-          result = player.run(playerArgs);
-          if (succeeded(result)) {
-            // Add to processed table list
-            processedTableList.add(new Pair<TableName, Path>(tableNames[i], bulkOutputPath));
-          } else {
-            throw new IOException("Can not merge backup images for " + dirs
-                + " (check Hadoop/MR and HBase logs). Player return code =" + result);
-          }
-          LOG.debug("Merge Job finished:" + result);
-        }
-        List<TableName> tableList = toTableNameList(processedTableList);
-        // PHASE 3
-        checkFailure(FailurePhase.PHASE3);
-        table.updateProcessedTablesForMerge(tableList);
-        finishedTables = true;
-
-        // Move data
-        for (Pair<TableName, Path> tn : processedTableList) {
-          moveData(fs, backupRoot, tn.getSecond(), tn.getFirst(), mergedBackupId);
-        }
-        // PHASE 4
-        checkFailure(FailurePhase.PHASE4);
-        // Delete old data and update manifest
-        List<String> backupsToDelete = getBackupIdsToDelete(backupIds, mergedBackupId);
-        deleteBackupImages(backupsToDelete, conn, fs, backupRoot);
-        updateBackupManifest(backupRoot, mergedBackupId, backupsToDelete);
-        // Finish merge session
-        table.finishMergeOperation();
-      } catch (RuntimeException e) {
-        throw e;
-      } catch (Exception e) {
-        LOG.error(e);
-        if (!finishedTables) {
-          // cleanup bulk directories and finish merge
-          // merge MUST be repeated (no need for repair)
-          cleanupBulkLoadDirs(fs, toPathList(processedTableList));
-          table.finishMergeOperation();
-          table.finishBackupExclusiveOperation();
-          throw new IOException("Backup merge operation failed, you should try it again", e);
-        } else {
-          // backup repair must be run
-          throw new IOException(
-              "Backup merge operation failed, run backup repair tool to restore system's integrity",
-              e);
-        }
-      } finally {
-        table.close();
-        conn.close();
-      }
-
-    }
-
-    private void checkFailure(FailurePhase phase) throws IOException {
-      if ( failurePhase != null && failurePhase == phase) {
-        throw new IOException (phase.toString());
-      }
-    }
-
-  }
-
-
-  @Test
-  public void TestIncBackupMergeRestore() throws Exception {
-
-    int ADD_ROWS = 99;
-    // #1 - create full backup for all tables
-    LOG.info("create full backup image for all tables");
-
-    List<TableName> tables = Lists.newArrayList(table1, table2);
-    // Set custom Merge Job implementation
-    conf1.setClass(BackupRestoreFactory.HBASE_BACKUP_MERGE_IMPL_CLASS,
-      BackupMergeJobWithFailures.class, BackupMergeJob.class);
-
-    Connection conn = ConnectionFactory.createConnection(conf1);
-
-    HBaseAdmin admin = null;
-    admin = (HBaseAdmin) conn.getAdmin();
-    BackupAdminImpl client = new BackupAdminImpl(conn);
-
-    BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR);
-    String backupIdFull = client.backupTables(request);
-
-    assertTrue(checkSucceeded(backupIdFull));
-
-    // #2 - insert some data to table1
-    HTable t1 = insertIntoTable(conn, table1, famName, 1, ADD_ROWS);
-    LOG.debug("writing " + ADD_ROWS + " rows to " + table1);
-
-    Assert.assertEquals(TEST_UTIL.countRows(t1), NB_ROWS_IN_BATCH + ADD_ROWS);
-    t1.close();
-    LOG.debug("written " + ADD_ROWS + " rows to " + table1);
-
-    HTable t2 = insertIntoTable(conn, table2, famName, 1, ADD_ROWS);
-
-    Assert.assertEquals(TEST_UTIL.countRows(t2), NB_ROWS_IN_BATCH + ADD_ROWS);
-    t2.close();
-    LOG.debug("written " + ADD_ROWS + " rows to " + table2);
-
-    // #3 - incremental backup for multiple tables
-    tables = Lists.newArrayList(table1, table2);
-    request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);
-    String backupIdIncMultiple = client.backupTables(request);
-
-    assertTrue(checkSucceeded(backupIdIncMultiple));
-
-    t1 = insertIntoTable(conn, table1, famName, 2, ADD_ROWS);
-    t1.close();
-
-    t2 = insertIntoTable(conn, table2, famName, 2, ADD_ROWS);
-    t2.close();
-
-    // #3 - incremental backup for multiple tables
-    request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);
-    String backupIdIncMultiple2 = client.backupTables(request);
-    assertTrue(checkSucceeded(backupIdIncMultiple2));
-
-    // #4 Merge backup images with failures
-
-    for ( FailurePhase phase : FailurePhase.values()) {
-      Configuration conf = conn.getConfiguration();
-
-      conf.set(FAILURE_PHASE_KEY, phase.toString());
-
-      try (BackupAdmin bAdmin = new BackupAdminImpl(conn);)
-      {
-        String[] backups = new String[] { backupIdIncMultiple, backupIdIncMultiple2 };
-        bAdmin.mergeBackups(backups);
-        Assert.fail("Expected IOException");
-      } catch (IOException e) {
-        BackupSystemTable table = new BackupSystemTable(conn);
-        if(phase.ordinal() < FailurePhase.PHASE4.ordinal()) {
-          // No need to repair:
-          // Both Merge and backup exclusive operations are finished
-          assertFalse(table.isMergeInProgress());
-          try {
-            table.finishBackupExclusiveOperation();
-            Assert.fail("IOException is expected");
-          } catch(IOException ee) {
-            // Expected
-          }
-        } else {
-          // Repair is required
-          assertTrue(table.isMergeInProgress());
-          try {
-            table.startBackupExclusiveOperation();
-            Assert.fail("IOException is expected");
-          } catch(IOException ee) {
-            // Expected - clean up before proceeding
-            table.finishMergeOperation();
-            table.finishBackupExclusiveOperation();
-          }
-        }
-        table.close();
-        LOG.debug("Expected :"+ e.getMessage());
-      }
-    }
-
-    // Now merge w/o failures
-    Configuration conf = conn.getConfiguration();
-    conf.unset(FAILURE_PHASE_KEY);
-    conf.unset(BackupRestoreFactory.HBASE_BACKUP_MERGE_IMPL_CLASS);
-
-    try (BackupAdmin bAdmin = new BackupAdminImpl(conn);) {
-      String[] backups = new String[] { backupIdIncMultiple, backupIdIncMultiple2 };
-      bAdmin.mergeBackups(backups);
-    }
-
-    // #6 - restore incremental backup for multiple tables, with overwrite
-    TableName[] tablesRestoreIncMultiple = new TableName[] { table1, table2 };
-    TableName[] tablesMapIncMultiple = new TableName[] { table1_restore, table2_restore };
-    client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupIdIncMultiple2, false,
-      tablesRestoreIncMultiple, tablesMapIncMultiple, true));
-
-    Table hTable = conn.getTable(table1_restore);
-    LOG.debug("After incremental restore: " + hTable.getTableDescriptor());
-    LOG.debug("f1 has " + TEST_UTIL.countRows(hTable, famName) + " rows");
-    Assert.assertEquals(TEST_UTIL.countRows(hTable, famName), NB_ROWS_IN_BATCH + 2 * ADD_ROWS);
-
-    hTable.close();
-
-    hTable = conn.getTable(table2_restore);
-    Assert.assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH + 2 * ADD_ROWS);
-    hTable.close();
-
-    admin.close();
-    conn.close();
-
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/37c65946/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithBulkLoad.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithBulkLoad.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithBulkLoad.java
deleted file mode 100644
index 769785f..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithBulkLoad.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import static org.junit.Assert.assertTrue;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl;
-import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
-import org.apache.hadoop.hbase.backup.util.BackupUtils;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.mapreduce.TestLoadIncrementalHFiles;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
-
-/**
- * 1. Create table t1
- * 2. Load data to t1
- * 3 Full backup t1
- * 4 Load data to t1
- * 5 bulk load into t1
- * 6 Incremental backup t1
- */
-@Category(LargeTests.class)
-@RunWith(Parameterized.class)
-public class TestIncrementalBackupWithBulkLoad extends TestBackupBase {
-  private static final Log LOG = LogFactory.getLog(TestIncrementalBackupDeleteTable.class);
-
-  @Parameterized.Parameters
-  public static Collection<Object[]> data() {
-    secure = true;
-    List<Object[]> params = new ArrayList<Object[]>();
-    params.add(new Object[] {Boolean.TRUE});
-    return params;
-  }
-
-  public TestIncrementalBackupWithBulkLoad(Boolean b) {
-  }
-  // implement all test cases in 1 test since incremental backup/restore has dependencies
-  @Test
-  public void TestIncBackupDeleteTable() throws Exception {
-    String testName = "TestIncBackupDeleteTable";
-    // #1 - create full backup for all tables
-    LOG.info("create full backup image for all tables");
-
-    List<TableName> tables = Lists.newArrayList(table1);
-    HBaseAdmin admin = null;
-    Connection conn = ConnectionFactory.createConnection(conf1);
-    admin = (HBaseAdmin) conn.getAdmin();
-    BackupAdminImpl client = new BackupAdminImpl(conn);
-
-    BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR);
-    String backupIdFull = client.backupTables(request);
-
-    assertTrue(checkSucceeded(backupIdFull));
-
-    // #2 - insert some data to table table1
-    HTable t1 = (HTable) conn.getTable(table1);
-    Put p1;
-    for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
-      p1 = new Put(Bytes.toBytes("row-t1" + i));
-      p1.addColumn(famName, qualName, Bytes.toBytes("val" + i));
-      t1.put(p1);
-    }
-
-    Assert.assertEquals(TEST_UTIL.countRows(t1), NB_ROWS_IN_BATCH * 2);
-    t1.close();
-
-    int NB_ROWS2 = 20;
-    LOG.debug("bulk loading into " + testName);
-    int actual = TestLoadIncrementalHFiles.loadHFiles(testName, table1Desc, TEST_UTIL, famName,
-        qualName, false, null, new byte[][][] {
-      new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("cccc") },
-      new byte[][]{ Bytes.toBytes("ddd"), Bytes.toBytes("ooo") },
-    }, true, false, true, NB_ROWS_IN_BATCH*2, NB_ROWS2);
-
-    // #3 - incremental backup for table1
-    tables = Lists.newArrayList(table1);
-    request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);
-    String backupIdIncMultiple = client.backupTables(request);
-    assertTrue(checkSucceeded(backupIdIncMultiple));
-
-    // #5.1 - check tables for full restore */
-    HBaseAdmin hAdmin = TEST_UTIL.getHBaseAdmin();
-
-    // #6 - restore incremental backup for table1
-    TableName[] tablesRestoreIncMultiple = new TableName[] { table1 };
-    TableName[] tablesMapIncMultiple = new TableName[] { table1_restore };
-    client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupIdIncMultiple,
-      false, tablesRestoreIncMultiple, tablesMapIncMultiple, true));
-
-    HTable hTable = (HTable) conn.getTable(table1_restore);
-    Assert.assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH * 2+actual);
-    request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR);
-
-    backupIdFull = client.backupTables(request);
-    try (final BackupSystemTable table = new BackupSystemTable(conn)) {
-      Pair<Map<TableName, Map<String, Map<String, List<Pair<String, Boolean>>>>>, List<byte[]>> pair
-      = table.readBulkloadRows(tables);
-      assertTrue("map still has " + pair.getSecond().size() + " entries",
-          pair.getSecond().isEmpty());
-    }
-    assertTrue(checkSucceeded(backupIdFull));
-
-    hTable.close();
-    admin.close();
-    conn.close();
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/37c65946/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithFailures.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithFailures.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithFailures.java
deleted file mode 100644
index 84a596e..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithFailures.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
-import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl;
-import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
-import org.apache.hadoop.hbase.backup.impl.TableBackupClient;
-import org.apache.hadoop.hbase.backup.impl.TableBackupClient.Stage;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.util.ToolRunner;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
-
-@Category(LargeTests.class)
-@RunWith(Parameterized.class)
-public class TestIncrementalBackupWithFailures extends TestBackupBase {
-  private static final Log LOG = LogFactory.getLog(TestIncrementalBackupWithFailures.class);
-
-  @Parameterized.Parameters
-  public static Collection<Object[]> data() {
-    provider = "multiwal";
-    List<Object[]> params = new ArrayList<Object[]>();
-    params.add(new Object[] { Boolean.TRUE });
-    return params;
-  }
-
-  public TestIncrementalBackupWithFailures(Boolean b) {
-  }
-
-  // implement all test cases in 1 test since incremental backup/restore has dependencies
-  @Test
-  public void testIncBackupRestore() throws Exception {
-
-    int ADD_ROWS = 99;
-    // #1 - create full backup for all tables
-    LOG.info("create full backup image for all tables");
-
-    List<TableName> tables = Lists.newArrayList(table1, table2);
-    final byte[] fam3Name = Bytes.toBytes("f3");
-    table1Desc.addFamily(new HColumnDescriptor(fam3Name));
-    HBaseTestingUtility.modifyTableSync(TEST_UTIL.getAdmin(), table1Desc);
-
-    Connection conn = ConnectionFactory.createConnection(conf1);
-    int NB_ROWS_FAM3 = 6;
-    insertIntoTable(conn, table1, fam3Name, 3, NB_ROWS_FAM3).close();
-
-    HBaseAdmin admin = null;
-    admin = (HBaseAdmin) conn.getAdmin();
-    BackupAdminImpl client = new BackupAdminImpl(conn);
-
-    BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR);
-    String backupIdFull = client.backupTables(request);
-
-    assertTrue(checkSucceeded(backupIdFull));
-
-    // #2 - insert some data to table
-    HTable t1 = insertIntoTable(conn, table1, famName, 1, ADD_ROWS);
-    LOG.debug("writing " + ADD_ROWS + " rows to " + table1);
-
-    Assert.assertEquals(TEST_UTIL.countRows(t1), NB_ROWS_IN_BATCH + ADD_ROWS + NB_ROWS_FAM3);
-    t1.close();
-    LOG.debug("written " + ADD_ROWS + " rows to " + table1);
-
-    HTable t2 = (HTable) conn.getTable(table2);
-    Put p2;
-    for (int i = 0; i < 5; i++) {
-      p2 = new Put(Bytes.toBytes("row-t2" + i));
-      p2.addColumn(famName, qualName, Bytes.toBytes("val" + i));
-      t2.put(p2);
-    }
-
-    Assert.assertEquals(TEST_UTIL.countRows(t2), NB_ROWS_IN_BATCH + 5);
-    t2.close();
-    LOG.debug("written " + 5 + " rows to " + table2);
-
-    // #3 - incremental backup for multiple tables
-    incrementalBackupWithFailures();
-
-    admin.close();
-    conn.close();
-
-  }
-
-
-  private void incrementalBackupWithFailures() throws Exception {
-    conf1.set(TableBackupClient.BACKUP_CLIENT_IMPL_CLASS,
-      IncrementalTableBackupClientForTest.class.getName());
-    int maxStage = Stage.values().length -1;
-    // Fail stages between 0 and 4 inclusive
-    for (int stage = 0; stage <= maxStage; stage++) {
-      LOG.info("Running stage " + stage);
-      runBackupAndFailAtStage(stage);
-    }
-  }
-
-  private void runBackupAndFailAtStage(int stage) throws Exception {
-
-    conf1.setInt(FullTableBackupClientForTest.BACKUP_TEST_MODE_STAGE, stage);
-    try (BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection())) {
-      int before = table.getBackupHistory().size();
-      String[] args =
-          new String[] { "create", "incremental", BACKUP_ROOT_DIR, "-t",
-              table1.getNameAsString() + "," + table2.getNameAsString() };
-      // Run backup
-      int ret = ToolRunner.run(conf1, new BackupDriver(), args);
-      assertFalse(ret == 0);
-      List<BackupInfo> backups = table.getBackupHistory();
-      int after = table.getBackupHistory().size();
-
-      assertTrue(after ==  before +1);
-      for (BackupInfo data : backups) {
-        if(data.getType() == BackupType.FULL) {
-          assertTrue(data.getState() == BackupState.COMPLETE);
-        } else {
-          assertTrue(data.getState() == BackupState.FAILED);
-        }
-      }
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/37c65946/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java
deleted file mode 100644
index 36a9ee2..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
- * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
- * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
- * for the specific language governing permissions and limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.concurrent.CountDownLatch;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.util.BackupUtils;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.snapshot.MobSnapshotTestingUtils;
-import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
-
-@Category(LargeTests.class)
-public class TestRemoteBackup extends TestBackupBase {
-
-  private static final Log LOG = LogFactory.getLog(TestRemoteBackup.class);
-
-  @Override
-  public void setUp () throws Exception {
-    useSecondCluster = true;
-    super.setUp();
-  }
-
-  /**
-   * Verify that a remote full backup is created on a single table with data correctly.
-   * @throws Exception
-   */
-  @Test
-  public void testFullBackupRemote() throws Exception {
-    LOG.info("test remote full backup on a single table");
-    final CountDownLatch latch = new CountDownLatch(1);
-    final int NB_ROWS_IN_FAM3 = 6;
-    final byte[] fam3Name = Bytes.toBytes("f3");
-    final byte[] fam2Name = Bytes.toBytes("f2");
-    final Connection conn = ConnectionFactory.createConnection(conf1);
-    Thread t = new Thread() {
-      @Override
-      public void run() {
-        try {
-          latch.await();
-        } catch (InterruptedException ie) {
-        }
-        try {
-          HTable t1 = (HTable) conn.getTable(table1);
-          Put p1;
-          for (int i = 0; i < NB_ROWS_IN_FAM3; i++) {
-            p1 = new Put(Bytes.toBytes("row-t1" + i));
-            p1.addColumn(fam3Name, qualName, Bytes.toBytes("val" + i));
-            t1.put(p1);
-          }
-          LOG.debug("Wrote " + NB_ROWS_IN_FAM3 + " rows into family3");
-          t1.close();
-        } catch (IOException ioe) {
-          throw new RuntimeException(ioe);
-        }
-      }
-    };
-    t.start();
-
-    table1Desc.addFamily(new HColumnDescriptor(fam3Name));
-    // family 2 is MOB enabled
-    HColumnDescriptor hcd = new HColumnDescriptor(fam2Name);
-    hcd.setMobEnabled(true);
-    hcd.setMobThreshold(0L);
-    table1Desc.addFamily(hcd);
-    HBaseTestingUtility.modifyTableSync(TEST_UTIL.getAdmin(), table1Desc);
-
-    SnapshotTestingUtils.loadData(TEST_UTIL, table1, 50, fam2Name);
-    HTable t1 = (HTable) conn.getTable(table1);
-    int rows0 = MobSnapshotTestingUtils.countMobRows(t1, fam2Name);
-
-    latch.countDown();
-    String backupId =
-        backupTables(BackupType.FULL, Lists.newArrayList(table1), BACKUP_REMOTE_ROOT_DIR);
-    assertTrue(checkSucceeded(backupId));
-
-    LOG.info("backup complete " + backupId);
-    Assert.assertEquals(TEST_UTIL.countRows(t1, famName), NB_ROWS_IN_BATCH);
-
-    t.join();
-    Assert.assertEquals(TEST_UTIL.countRows(t1, fam3Name), NB_ROWS_IN_FAM3);
-    t1.close();
-
-    TableName[] tablesRestoreFull = new TableName[] { table1 };
-
-    TableName[] tablesMapFull = new TableName[] { table1_restore };
-
-    BackupAdmin client = getBackupAdmin();
-    client.restore(BackupUtils.createRestoreRequest(BACKUP_REMOTE_ROOT_DIR, backupId, false,
-      tablesRestoreFull, tablesMapFull, false));
-
-    // check tables for full restore
-    HBaseAdmin hAdmin = TEST_UTIL.getHBaseAdmin();
-    assertTrue(hAdmin.tableExists(table1_restore));
-
-    // #5.2 - checking row count of tables for full restore
-    HTable hTable = (HTable) conn.getTable(table1_restore);
-    Assert.assertEquals(TEST_UTIL.countRows(hTable, famName), NB_ROWS_IN_BATCH);
-    int cnt3 = TEST_UTIL.countRows(hTable, fam3Name);
-    Assert.assertTrue(cnt3 >= 0 && cnt3 <= NB_ROWS_IN_FAM3);
-
-    int rows1 = MobSnapshotTestingUtils.countMobRows(t1, fam2Name);
-    Assert.assertEquals(rows0, rows1);
-    hTable.close();
-
-    hAdmin.close();
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hbase/blob/37c65946/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java
deleted file mode 100644
index 0386c27..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
- * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
- * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
- * for the specific language governing permissions and limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import static org.junit.Assert.assertTrue;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.util.BackupUtils;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category(LargeTests.class)
-public class TestRemoteRestore extends TestBackupBase {
-
-  private static final Log LOG = LogFactory.getLog(TestRemoteRestore.class);
-
-  @Override
-  public void setUp () throws Exception {
-    useSecondCluster = true;
-    super.setUp();
-  }
-
-
-  /**
-   * Verify that a remote restore on a single table is successful.
-   * @throws Exception
-   */
-  @Test
-  public void testFullRestoreRemote() throws Exception {
-
-    LOG.info("test remote full backup on a single table");
-    String backupId =
-        backupTables(BackupType.FULL, toList(table1.getNameAsString()), BACKUP_REMOTE_ROOT_DIR);
-    LOG.info("backup complete");
-    TableName[] tableset = new TableName[] { table1 };
-    TableName[] tablemap = new TableName[] { table1_restore };
-    getBackupAdmin().restore(
-      BackupUtils.createRestoreRequest(BACKUP_REMOTE_ROOT_DIR, backupId, false, tableset,
-        tablemap, false));
-    HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
-    assertTrue(hba.tableExists(table1_restore));
-    TEST_UTIL.deleteTable(table1_restore);
-    hba.close();
-  }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hbase/blob/37c65946/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRepairAfterFailedDelete.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRepairAfterFailedDelete.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRepairAfterFailedDelete.java
deleted file mode 100644
index 556521f..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRepairAfterFailedDelete.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import static org.junit.Assert.assertTrue;
-
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.util.ToolRunner;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
-
-@Category(LargeTests.class)
-public class TestRepairAfterFailedDelete extends TestBackupBase {
-
-  private static final Log LOG = LogFactory.getLog(TestRepairAfterFailedDelete.class);
-
-  @Test
-  public void testRepairBackupDelete() throws Exception {
-    LOG.info("test repair backup delete on a single table with data");
-    List<TableName> tableList = Lists.newArrayList(table1);
-    String backupId = fullTableBackup(tableList);
-    assertTrue(checkSucceeded(backupId));
-    LOG.info("backup complete");
-    String[] backupIds = new String[] { backupId };
-    BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection());
-    BackupInfo info = table.readBackupInfo(backupId);
-    Path path = new Path(info.getBackupRootDir(), backupId);
-    FileSystem fs = FileSystem.get(path.toUri(), conf1);
-    assertTrue(fs.exists(path));
-
-    // Snapshot backup system table before delete
-    String snapshotName = "snapshot-backup";
-    Connection conn = TEST_UTIL.getConnection();
-    Admin admin = conn.getAdmin();
-    admin.snapshot(snapshotName, BackupSystemTable.getTableName(conf1));
-
-    int deleted = getBackupAdmin().deleteBackups(backupIds);
-
-    assertTrue(!fs.exists(path));
-    assertTrue(fs.exists(new Path(info.getBackupRootDir())));
-    assertTrue(1 == deleted);
-
-    // Emulate delete failure
-    // Restore backup system table
-    admin.disableTable(BackupSystemTable.getTableName(conf1));
-    admin.restoreSnapshot(snapshotName);
-    admin.enableTable(BackupSystemTable.getTableName(conf1));
-    // Start backup session
-    table.startBackupExclusiveOperation();
-    // Start delete operation
-    table.startDeleteOperation(backupIds);
-
-    // Now run repair command to repair "failed" delete operation
-    String[] args = new String[] {"repair"};
-    // Run restore
-    int ret = ToolRunner.run(conf1, new BackupDriver(), args);
-    assertTrue(ret == 0);
-    // Verify that history length == 0
-    assertTrue (table.getBackupHistory().size() == 0);
-    table.close();
-    admin.close();
-  }
-
-
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/37c65946/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java
deleted file mode 100644
index c61b018..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import static org.junit.Assert.assertTrue;
-
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.util.BackupUtils;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category(LargeTests.class)
-public class TestRestoreBoundaryTests extends TestBackupBase {
-
-  private static final Log LOG = LogFactory.getLog(TestRestoreBoundaryTests.class);
-
-  /**
-   * Verify that a single empty table is restored to a new table
-   * @throws Exception
-   */
-  @Test
-  public void testFullRestoreSingleEmpty() throws Exception {
-    LOG.info("test full restore on a single table empty table");
-    String backupId = fullTableBackup(toList(table1.getNameAsString()));
-    LOG.info("backup complete");
-    TableName[] tableset = new TableName[] { table1 };
-    TableName[] tablemap = new TableName[] { table1_restore };
-    getBackupAdmin().restore(
-      BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, tableset, tablemap,
-        false));
-    HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
-    assertTrue(hba.tableExists(table1_restore));
-    TEST_UTIL.deleteTable(table1_restore);
-  }
-
-  /**
-   * Verify that multiple tables are restored to new tables.
-   * @throws Exception
-   */
-  @Test
-  public void testFullRestoreMultipleEmpty() throws Exception {
-    LOG.info("create full backup image on multiple tables");
-
-    List<TableName> tables = toList(table2.getNameAsString(), table3.getNameAsString());
-    String backupId = fullTableBackup(tables);
-    TableName[] restore_tableset = new TableName[] { table2, table3 };
-    TableName[] tablemap = new TableName[] { table2_restore, table3_restore };
-    getBackupAdmin().restore(
-      BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, restore_tableset,
-        tablemap, false));
-    HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
-    assertTrue(hba.tableExists(table2_restore));
-    assertTrue(hba.tableExists(table3_restore));
-    TEST_UTIL.deleteTable(table2_restore);
-    TEST_UTIL.deleteTable(table3_restore);
-    hba.close();
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hbase/blob/37c65946/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestSystemTableSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestSystemTableSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestSystemTableSnapshot.java
deleted file mode 100644
index 6443421..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestSystemTableSnapshot.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.junit.experimental.categories.Category;
-
-@Category(LargeTests.class)
-public class TestSystemTableSnapshot extends TestBackupBase {
-
-  private static final Log LOG = LogFactory.getLog(TestSystemTableSnapshot.class);
-
-  /**
-   * Verify backup system table snapshot
-   * @throws Exception
-   */
- // @Test
-  public void _testBackupRestoreSystemTable() throws Exception {
-
-    LOG.info("test snapshot system table");
-
-    TableName backupSystem = BackupSystemTable.getTableName(conf1);
-
-    HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
-    String snapshotName = "sysTable";
-    hba.snapshot(snapshotName, backupSystem);
-
-    hba.disableTable(backupSystem);
-    hba.restoreSnapshot(snapshotName);
-    hba.enableTable(backupSystem);
-    hba.close();
-  }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hbase/blob/37c65946/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java
deleted file mode 100644
index 5f72f45..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup.master;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocatedFileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.BackupType;
-import org.apache.hadoop.hbase.backup.TestBackupBase;
-import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-import org.apache.hadoop.hbase.shaded.com.google.common.collect.Iterables;
-import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
-
-@Category(LargeTests.class)
-public class TestBackupLogCleaner extends TestBackupBase {
-  private static final Log LOG = LogFactory.getLog(TestBackupLogCleaner.class);
-
-  // implements all test cases in 1 test since incremental full backup/
-  // incremental backup has dependencies
-  @Test
-  public void testBackupLogCleaner() throws Exception {
-
-    // #1 - create full backup for all tables
-    LOG.info("create full backup image for all tables");
-
-    List<TableName> tableSetFullList = Lists.newArrayList(table1, table2, table3, table4);
-
-    try (BackupSystemTable systemTable = new BackupSystemTable(TEST_UTIL.getConnection())) {
-      // Verify that we have no backup sessions yet
-      assertFalse(systemTable.hasBackupSessions());
-
-      List<FileStatus> walFiles = getListOfWALFiles(TEST_UTIL.getConfiguration());
-      List<String> swalFiles = convert(walFiles);
-      BackupLogCleaner cleaner = new BackupLogCleaner();
-      cleaner.setConf(TEST_UTIL.getConfiguration());
-      cleaner.init(null);
-      cleaner.setConf(TEST_UTIL.getConfiguration());
-
-      Iterable<FileStatus> deletable = cleaner.getDeletableFiles(walFiles);
-      int size = Iterables.size(deletable);
-
-      // We can delete all files because we do not have yet recorded backup sessions
-      assertTrue(size == walFiles.size());
-
-      systemTable.addWALFiles(swalFiles, "backup", "root");
-      String backupIdFull = fullTableBackup(tableSetFullList);
-      assertTrue(checkSucceeded(backupIdFull));
-      // Check one more time
-      deletable = cleaner.getDeletableFiles(walFiles);
-      // We can delete wal files because they were saved into backup system table table
-      size = Iterables.size(deletable);
-      assertTrue(size == walFiles.size());
-
-      List<FileStatus> newWalFiles = getListOfWALFiles(TEST_UTIL.getConfiguration());
-      LOG.debug("WAL list after full backup");
-      convert(newWalFiles);
-
-      // New list of wal files is greater than the previous one,
-      // because new wal per RS have been opened after full backup
-      assertTrue(walFiles.size() < newWalFiles.size());
-      Connection conn = ConnectionFactory.createConnection(conf1);
-      // #2 - insert some data to table
-      HTable t1 = (HTable) conn.getTable(table1);
-      Put p1;
-      for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
-        p1 = new Put(Bytes.toBytes("row-t1" + i));
-        p1.addColumn(famName, qualName, Bytes.toBytes("val" + i));
-        t1.put(p1);
-      }
-
-      t1.close();
-
-      HTable t2 = (HTable) conn.getTable(table2);
-      Put p2;
-      for (int i = 0; i < 5; i++) {
-        p2 = new Put(Bytes.toBytes("row-t2" + i));
-        p2.addColumn(famName, qualName, Bytes.toBytes("val" + i));
-        t2.put(p2);
-      }
-
-      t2.close();
-
-      // #3 - incremental backup for multiple tables
-
-      List<TableName> tableSetIncList = Lists.newArrayList(table1, table2, table3);
-      String backupIdIncMultiple = backupTables(BackupType.INCREMENTAL, tableSetIncList,
-        BACKUP_ROOT_DIR);
-      assertTrue(checkSucceeded(backupIdIncMultiple));
-      deletable = cleaner.getDeletableFiles(newWalFiles);
-
-      assertTrue(Iterables.size(deletable) == newWalFiles.size());
-
-      conn.close();
-    }
-  }
-
-  private List<String> convert(List<FileStatus> walFiles) {
-    List<String> result = new ArrayList<String>();
-    for (FileStatus fs : walFiles) {
-      LOG.debug("+++WAL: " + fs.getPath().toString());
-      result.add(fs.getPath().toString());
-    }
-    return result;
-  }
-
-  private List<FileStatus> getListOfWALFiles(Configuration c) throws IOException {
-    Path logRoot = new Path(FSUtils.getRootDir(c), HConstants.HREGION_LOGDIR_NAME);
-    FileSystem fs = FileSystem.get(c);
-    RemoteIterator<LocatedFileStatus> it = fs.listFiles(logRoot, true);
-    List<FileStatus> logFiles = new ArrayList<FileStatus>();
-    while (it.hasNext()) {
-      LocatedFileStatus lfs = it.next();
-      if (lfs.isFile() && !AbstractFSWALProvider.isMetaFile(lfs.getPath())) {
-        logFiles.add(lfs);
-        LOG.info(lfs);
-      }
-    }
-    return logFiles;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/37c65946/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 484ba19..0ee9ba0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -87,6 +87,7 @@
     <module>hbase-metrics-api</module>
     <module>hbase-metrics</module>
     <module>hbase-spark-it</module>
+    <module>hbase-backup</module>
   </modules>
   <!--Add apache snapshots in case we want to use unreleased versions of plugins:
       e.g. surefire 2.18-SNAPSHOT-->


Mime
View raw message