hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mberto...@apache.org
Subject [2/6] hbase git commit: HBASE-16649 Truncate table with splits preserved can cause both data loss and truncated data appeared again
Date Mon, 26 Sep 2016 20:49:55 GMT
HBASE-16649 Truncate table with splits preserved can cause both data loss and truncated data
appeared again


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4566e4df
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4566e4df
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4566e4df

Branch: refs/heads/branch-1
Commit: 4566e4df58bdd176228aab2cd3cfd80dd983072f
Parents: d6f9eab
Author: Matteo Bertozzi <matteo.bertozzi@cloudera.com>
Authored: Mon Sep 26 12:58:04 2016 -0700
Committer: Matteo Bertozzi <matteo.bertozzi@cloudera.com>
Committed: Mon Sep 26 13:09:06 2016 -0700

----------------------------------------------------------------------
 .../hadoop/hbase/master/CatalogJanitor.java       |  8 +++++---
 .../apache/hadoop/hbase/master/ServerManager.java | 18 ++++++++++++++++++
 .../master/procedure/DeleteTableProcedure.java    |  3 +++
 .../master/procedure/TruncateTableProcedure.java  | 10 ++++++++++
 .../hadoop/hbase/master/TestCatalogJanitor.java   |  4 +++-
 .../procedure/TestTruncateTableProcedure.java     |  8 ++++----
 6 files changed, 43 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/4566e4df/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index a921112..17644eb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -216,8 +216,9 @@ public class CatalogJanitor extends ScheduledChore {
           + " from fs because merged region no longer holds references");
       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionA);
       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionB);
-      MetaTableAccessor.deleteMergeQualifiers(server.getConnection(),
-        mergedRegion);
+      MetaTableAccessor.deleteMergeQualifiers(services.getConnection(), mergedRegion);
+      services.getServerManager().removeRegion(regionA);
+      services.getServerManager().removeRegion(regionB);
       return true;
     }
     return false;
@@ -360,6 +361,7 @@ public class CatalogJanitor extends ScheduledChore {
       if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent);
       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent);
       MetaTableAccessor.deleteRegion(this.connection, parent);
+      services.getServerManager().removeRegion(parent);
       result = true;
     }
     return result;
@@ -461,4 +463,4 @@ public class CatalogJanitor extends ScheduledChore {
     return cleanMergeRegion(region, mergeRegions.getFirst(),
         mergeRegions.getSecond());
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/4566e4df/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 5eb569e..bcaa4d0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -1212,4 +1212,22 @@ public class ServerManager {
       deadservers.cleanAllPreviousInstances(serverName);
     }
   }
+
+  /**
+   * Called by delete table and similar to notify the ServerManager that a region was removed.
+   */
+  public void removeRegion(final HRegionInfo regionInfo) {
+    final byte[] encodedName = regionInfo.getEncodedNameAsBytes();
+    storeFlushedSequenceIdsByRegion.remove(encodedName);
+    flushedSequenceIdByRegion.remove(encodedName);
+  }
+
+  /**
+   * Called by delete table and similar to notify the ServerManager that a region was removed.
+   */
+  public void removeRegions(final List<HRegionInfo> regions) {
+    for (HRegionInfo hri: regions) {
+      removeRegion(hri);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/4566e4df/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
index 44dde9e..20466d1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
@@ -379,6 +379,9 @@ public class DeleteTableProcedure
 
     // Clean any remaining rows for this table.
     cleanAnyRemainingRows(env, tableName);
+
+    // clean region references from the server manager
+    env.getMasterServices().getServerManager().removeRegions(regions);
   }
 
   protected static void deleteAssignmentState(final MasterProcedureEnv env,

http://git-wip-us.apache.org/repos/asf/hbase/blob/4566e4df/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
index 3a3739d..9dccef6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
@@ -105,6 +105,8 @@ public class TruncateTableProcedure
           if (!preserveSplits) {
             // if we are not preserving splits, generate a new single region
             regions = Arrays.asList(ModifyRegionUtils.createHRegionInfos(hTableDescriptor,
null));
+          } else {
+            regions = recreateRegionInfo(regions);
           }
           setNextState(TruncateTableState.TRUNCATE_TABLE_CREATE_FS_LAYOUT);
           break;
@@ -245,6 +247,14 @@ public class TruncateTableProcedure
     }
   }
 
+  private static List<HRegionInfo> recreateRegionInfo(final List<HRegionInfo>
regions) {
+    ArrayList<HRegionInfo> newRegions = new ArrayList<HRegionInfo>(regions.size());
+    for (HRegionInfo hri: regions) {
+      newRegions.add(new HRegionInfo(hri.getTable(), hri.getStartKey(), hri.getEndKey()));
+    }
+    return newRegions;
+  }
+
   private boolean prepareTruncate(final MasterProcedureEnv env) throws IOException {
     try {
       env.getMasterServices().checkTableModifiable(getTableName());

http://git-wip-us.apache.org/repos/asf/hbase/blob/4566e4df/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index 1fc74bc..13c0cb1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -216,10 +216,12 @@ public class TestCatalogJanitor {
   class MockMasterServices implements MasterServices {
     private final MasterFileSystem mfs;
     private final AssignmentManager asm;
+    private final ServerManager sm;
 
     MockMasterServices(final Server server) throws IOException {
       this.mfs = new MasterFileSystem(server, this);
       this.asm = Mockito.mock(AssignmentManager.class);
+      this.sm = Mockito.mock(ServerManager.class);
     }
 
     @Override
@@ -289,7 +291,7 @@ public class TestCatalogJanitor {
 
     @Override
     public ServerManager getServerManager() {
-      return null;
+      return sm;
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/hbase/blob/4566e4df/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
index 08442ce..941dd2f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
@@ -162,10 +162,10 @@ public class TestTruncateTableProcedure {
     UTIL.waitUntilAllRegionsAssigned(tableName);
 
     // validate the table regions and layout
+    regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[0]);
     if (preserveSplits) {
-      assertEquals(1 + splitKeys.length, UTIL.getHBaseAdmin().getTableRegions(tableName).size());
+      assertEquals(1 + splitKeys.length, regions.length);
     } else {
-      regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[1]);
       assertEquals(1, regions.length);
     }
     MasterProcedureTestingUtility.validateTableCreation(
@@ -229,10 +229,10 @@ public class TestTruncateTableProcedure {
     UTIL.waitUntilAllRegionsAssigned(tableName);
 
     // validate the table regions and layout
+    regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[0]);
     if (preserveSplits) {
-      assertEquals(1 + splitKeys.length, UTIL.getHBaseAdmin().getTableRegions(tableName).size());
+      assertEquals(1 + splitKeys.length, regions.length);
     } else {
-      regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[1]);
       assertEquals(1, regions.length);
     }
     MasterProcedureTestingUtility.validateTableCreation(


Mime
View raw message