hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mberto...@apache.org
Subject [5/6] hbase git commit: HBASE-16649 Truncate table with splits preserved can cause both data loss and truncated data appeared again
Date Mon, 26 Sep 2016 20:49:58 GMT
HBASE-16649 Truncate table with splits preserved can cause both data loss and truncated data
appeared again


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/88512be5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/88512be5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/88512be5

Branch: refs/heads/branch-1.1
Commit: 88512be52b8707fb87ab2c5979fd71664a417a90
Parents: cdc799e
Author: Matteo Bertozzi <matteo.bertozzi@cloudera.com>
Authored: Mon Sep 26 12:58:04 2016 -0700
Committer: Matteo Bertozzi <matteo.bertozzi@cloudera.com>
Committed: Mon Sep 26 13:21:24 2016 -0700

----------------------------------------------------------------------
 .../hadoop/hbase/master/CatalogJanitor.java       |  8 +++++---
 .../apache/hadoop/hbase/master/ServerManager.java | 18 ++++++++++++++++++
 .../master/handler/TruncateTableHandler.java      | 11 ++++++++++-
 .../master/procedure/DeleteTableProcedure.java    |  3 +++
 .../master/procedure/TruncateTableProcedure.java  | 10 ++++++++++
 .../hadoop/hbase/master/TestCatalogJanitor.java   |  4 +++-
 .../procedure/TestTruncateTableProcedure.java     |  8 ++++----
 7 files changed, 53 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/88512be5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index d686d7c..48105c8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -215,8 +215,9 @@ public class CatalogJanitor extends ScheduledChore {
           + " from fs because merged region no longer holds references");
       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionA);
       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionB);
-      MetaTableAccessor.deleteMergeQualifiers(server.getConnection(),
-        mergedRegion);
+      MetaTableAccessor.deleteMergeQualifiers(services.getConnection(), mergedRegion);
+      services.getServerManager().removeRegion(regionA);
+      services.getServerManager().removeRegion(regionB);
       return true;
     }
     return false;
@@ -349,6 +350,7 @@ public class CatalogJanitor extends ScheduledChore {
       if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent);
       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent);
       MetaTableAccessor.deleteRegion(this.connection, parent);
+      services.getServerManager().removeRegion(parent);
       result = true;
     }
     return result;
@@ -450,4 +452,4 @@ public class CatalogJanitor extends ScheduledChore {
     return cleanMergeRegion(region, mergeRegions.getFirst(),
         mergeRegions.getSecond());
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/88512be5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 11ca461..02ed231 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -1192,4 +1192,22 @@ public class ServerManager {
       deadservers.cleanAllPreviousInstances(serverName);
     }
   }
+
+  /**
+   * Called by delete table and similar to notify the ServerManager that a region was removed.
+   */
+  public void removeRegion(final HRegionInfo regionInfo) {
+    final byte[] encodedName = regionInfo.getEncodedNameAsBytes();
+    storeFlushedSequenceIdsByRegion.remove(encodedName);
+    flushedSequenceIdByRegion.remove(encodedName);
+  }
+
+  /**
+   * Called by delete table and similar to notify the ServerManager that a region was removed.
+   */
+  public void removeRegions(final List<HRegionInfo> regions) {
+    for (HRegionInfo hri: regions) {
+      removeRegion(hri);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/88512be5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java
index 67bf343..600ae44 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java
@@ -104,7 +104,7 @@ public class TruncateTableHandler extends DeleteTableHandler {
 
       HRegionInfo[] newRegions;
       if (this.preserveSplits) {
-        newRegions = regions.toArray(new HRegionInfo[regions.size()]);
+        newRegions = recreateRegionInfo(regions);
         LOG.info("Truncate will preserve " + newRegions.length + " regions");
       } else {
         newRegions = new HRegionInfo[1];
@@ -143,4 +143,13 @@ public class TruncateTableHandler extends DeleteTableHandler {
       throw e;
     }
   }
+
+  private static HRegionInfo[] recreateRegionInfo(final List<HRegionInfo> regions)
{
+    HRegionInfo[] newRegions = new HRegionInfo[regions.size()];
+    int index = 0;
+    for (HRegionInfo hri: regions) {
+      newRegions[index++] = new HRegionInfo(hri.getTable(), hri.getStartKey(), hri.getEndKey());
+    }
+    return newRegions;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/88512be5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
index 45cb2eb..734fa13 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
@@ -381,6 +381,9 @@ public class DeleteTableProcedure
 
     // Clean any remaining rows for this table.
     cleanAnyRemainingRows(env, tableName);
+
+    // clean region references from the server manager
+    env.getMasterServices().getServerManager().removeRegions(regions);
   }
 
   protected static void deleteAssignmentState(final MasterProcedureEnv env,

http://git-wip-us.apache.org/repos/asf/hbase/blob/88512be5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
index 1df18f4..d065d2f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
@@ -104,6 +104,8 @@ public class TruncateTableProcedure
           if (!preserveSplits) {
             // if we are not preserving splits, generate a new single region
             regions = Arrays.asList(ModifyRegionUtils.createHRegionInfos(hTableDescriptor,
null));
+          } else {
+            regions = recreateRegionInfo(regions);
           }
           setNextState(TruncateTableState.TRUNCATE_TABLE_CREATE_FS_LAYOUT);
           break;
@@ -247,6 +249,14 @@ public class TruncateTableProcedure
     }
   }
 
+  private static List<HRegionInfo> recreateRegionInfo(final List<HRegionInfo>
regions) {
+    ArrayList<HRegionInfo> newRegions = new ArrayList<HRegionInfo>(regions.size());
+    for (HRegionInfo hri: regions) {
+      newRegions.add(new HRegionInfo(hri.getTable(), hri.getStartKey(), hri.getEndKey()));
+    }
+    return newRegions;
+  }
+
   private boolean prepareTruncate(final MasterProcedureEnv env) throws IOException {
     try {
       env.getMasterServices().checkTableModifiable(getTableName());

http://git-wip-us.apache.org/repos/asf/hbase/blob/88512be5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index 1f9c087..ae175b9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -213,10 +213,12 @@ public class TestCatalogJanitor {
   class MockMasterServices implements MasterServices {
     private final MasterFileSystem mfs;
     private final AssignmentManager asm;
+    private final ServerManager sm;
 
     MockMasterServices(final Server server) throws IOException {
       this.mfs = new MasterFileSystem(server, this);
       this.asm = Mockito.mock(AssignmentManager.class);
+      this.sm = Mockito.mock(ServerManager.class);
     }
 
     @Override
@@ -276,7 +278,7 @@ public class TestCatalogJanitor {
 
     @Override
     public ServerManager getServerManager() {
-      return null;
+      return sm;
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/hbase/blob/88512be5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
index 08442ce..941dd2f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
@@ -162,10 +162,10 @@ public class TestTruncateTableProcedure {
     UTIL.waitUntilAllRegionsAssigned(tableName);
 
     // validate the table regions and layout
+    regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[0]);
     if (preserveSplits) {
-      assertEquals(1 + splitKeys.length, UTIL.getHBaseAdmin().getTableRegions(tableName).size());
+      assertEquals(1 + splitKeys.length, regions.length);
     } else {
-      regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[1]);
       assertEquals(1, regions.length);
     }
     MasterProcedureTestingUtility.validateTableCreation(
@@ -229,10 +229,10 @@ public class TestTruncateTableProcedure {
     UTIL.waitUntilAllRegionsAssigned(tableName);
 
     // validate the table regions and layout
+    regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[0]);
     if (preserveSplits) {
-      assertEquals(1 + splitKeys.length, UTIL.getHBaseAdmin().getTableRegions(tableName).size());
+      assertEquals(1 + splitKeys.length, regions.length);
     } else {
-      regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[1]);
       assertEquals(1, regions.length);
     }
     MasterProcedureTestingUtility.validateTableCreation(


Mime
View raw message