hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject svn commit: r1570008 - in /hbase/branches/hbase-10070: hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/ hbase-server/src/main/java/org/apache/hadoop/hbase/master/ hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ hbase-serve...
Date Thu, 20 Feb 2014 00:26:56 GMT
Author: ddas
Date: Thu Feb 20 00:26:56 2014
New Revision: 1570008

URL: http://svn.apache.org/r1570008
Log:
HBASE-10361. Enable/AlterTable support for region replicas (ddas)

Modified:
    hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
    hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
    hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
    hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java
    hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java
    hbase/branches/hbase-10070/hbase-shell/src/main/ruby/hbase/admin.rb
    hbase/branches/hbase-10070/hbase-shell/src/main/ruby/shell/commands/alter.rb

Modified: hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java?rev=1570008&r1=1570007&r2=1570008&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
(original)
+++ hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
Thu Feb 20 00:26:56 2014
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -174,6 +175,31 @@ public class MetaEditor extends MetaRead
   }
 
   /**
+   * Deletes some replica columns corresponding to replicas for the passed rows
+   * @param metaRows
+   * @param replicaIndexToDeleteFrom the replica ID we would start deleting from
+   * @param numReplicasToRemove
+   * @param ct
+   * @throws IOException
+   */
+  public static void removeRegionReplicasFromMeta(Set<byte[]> metaRows, int replicaIndexToDeleteFrom,
+      int numReplicasToRemove, CatalogTracker ct) throws IOException {
+    int absoluteIndex = replicaIndexToDeleteFrom + numReplicasToRemove;
+    for (byte[] row : metaRows) {
+      Delete deleteReplicaLocations = new Delete(row);
+      for (int i = replicaIndexToDeleteFrom; i < absoluteIndex; i++) {
+        deleteReplicaLocations.deleteColumns(HConstants.CATALOG_FAMILY,
+            MetaReader.getServerColumn(i));
+        deleteReplicaLocations.deleteColumns(HConstants.CATALOG_FAMILY,
+            MetaReader.getSeqNumColumn(i));
+        deleteReplicaLocations.deleteColumns(HConstants.CATALOG_FAMILY,
+            MetaReader.getStartCodeColumn(i));
+      }
+      deleteFromMetaTable(ct, deleteReplicaLocations);
+    }
+  }
+
+  /**
    * Execute the passed <code>mutations</code> against <code>hbase:meta</code>
table.
    * @param ct CatalogTracker on whose back we will ride the edit.
    * @param mutations Puts and Deletes to execute on hbase:meta

Modified: hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java?rev=1570008&r1=1570007&r2=1570008&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
(original)
+++ hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
Thu Feb 20 00:26:56 2014
@@ -453,9 +453,11 @@ public class RegionStates {
     }
     HRegionInfo defaultReplica = RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
     Set<HRegionInfo> replicas = defaultReplicaToOtherReplicas.get(defaultReplica);
-    replicas.remove(hri);
-    if (replicas.isEmpty()) {
-      defaultReplicaToOtherReplicas.remove(defaultReplica);
+    if (replicas != null) {
+      replicas.remove(hri);
+      if (replicas.isEmpty()) {
+        defaultReplicaToOtherReplicas.remove(defaultReplica);
+      }
     }
   }
 

Modified: hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java?rev=1570008&r1=1570007&r2=1570008&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
(original)
+++ hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
Thu Feb 20 00:26:56 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.master.h
 
 import java.io.IOException;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutorService;
@@ -42,6 +43,7 @@ import org.apache.hadoop.hbase.master.Bu
 import org.apache.hadoop.hbase.master.GeneralBulkAssigner;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.RegionStates;
 import org.apache.hadoop.hbase.master.ServerManager;
 import org.apache.hadoop.hbase.master.TableLockManager;
@@ -62,6 +64,7 @@ public class EnableTableHandler extends 
   private final CatalogTracker catalogTracker;
   private boolean skipTableStateCheck = false;
   private TableLock tableLock;
+  private MasterServices services;
 
   public EnableTableHandler(Server server, TableName tableName,
       CatalogTracker catalogTracker, AssignmentManager assignmentManager,
@@ -74,6 +77,14 @@ public class EnableTableHandler extends 
     this.skipTableStateCheck = skipTableStateCheck;
   }
 
+  public EnableTableHandler(MasterServices services, TableName tableName,
+      CatalogTracker catalogTracker, AssignmentManager assignmentManager,
+      TableLockManager tableLockManager, boolean skipTableStateCheck) {
+    this((Server)services, tableName, catalogTracker, assignmentManager, tableLockManager,
+        skipTableStateCheck);
+    this.services = services;
+  }
+
   public EnableTableHandler prepare()
       throws TableNotFoundException, TableNotDisabledException, IOException {
     //acquire the table write lock, blocking
@@ -183,6 +194,16 @@ public class EnableTableHandler extends 
     int countOfRegionsInTable = tableRegionsAndLocations.size();
     Map<HRegionInfo, ServerName> regionsToAssign =
         regionsToAssignWithServerName(tableRegionsAndLocations);
+    if (services != null) {
+      // need to potentially create some regions for the replicas
+      List<HRegionInfo> unrecordedReplicas = AssignmentManager.replicaRegionsNotRecordedInMeta(
+          new HashSet<HRegionInfo>(regionsToAssign.keySet()), services);
+      for (HRegionInfo h : unrecordedReplicas) {
+        regionsToAssign.put(h, 
+            this.assignmentManager.getBalancer().randomAssignment(h,
+                serverManager.getOnlineServersList()));
+      }
+    }
     int regionsCount = regionsToAssign.size();
     if (regionsCount == 0) {
       done = true;

Modified: hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java?rev=1570008&r1=1570007&r2=1570008&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java
(original)
+++ hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java
Thu Feb 20 00:26:56 2014
@@ -19,16 +19,24 @@
 package org.apache.hadoop.hbase.master.handler;
 
 import java.io.IOException;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.executor.EventType;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
@@ -53,8 +61,12 @@ public class ModifyTableHandler extends 
   @Override
   protected void prepareWithTableLock() throws IOException {
     super.prepareWithTableLock();
-    // Check table exists.
-    getTableDescriptor();
+    // Check operation is possible on the table in its current state
+    // Also checks whether the table exists
+    if (masterServices.getAssignmentManager().getZKTable().isEnabledTable(this.htd.getTableName())
+        && this.htd.getRegionReplication() != getTableDescriptor().getRegionReplication())
{
+      throw new IOException("REGION_REPLICATION change is not supported for enabled tables");
+    } 
   }
 
   @Override
@@ -68,11 +80,35 @@ public class ModifyTableHandler extends 
     HTableDescriptor oldHtd = getTableDescriptor();
     this.masterServices.getTableDescriptors().add(this.htd);
     deleteFamilyFromFS(hris, oldHtd.getFamiliesKeys());
+    removeReplicaColumnsIfNeeded(this.htd.getRegionReplication(), oldHtd.getRegionReplication(),
+        htd.getTableName());
     if (cpHost != null) {
       cpHost.postModifyTableHandler(this.tableName, this.htd);
     }
   }
 
+  private void removeReplicaColumnsIfNeeded(int newReplicaCount, int oldReplicaCount,
+      TableName table) throws IOException {
+    if (newReplicaCount >= oldReplicaCount) return;
+    Set<byte[]> tableRows = new HashSet<byte[]>();
+    Scan scan = MetaReader.getScanForTableName(table);
+    scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+    HTable htable = null;
+    try {
+      htable = new HTable(masterServices.getConfiguration(), TableName.META_TABLE_NAME);
+      ResultScanner resScanner = htable.getScanner(scan);
+      for (Result result : resScanner) {
+        tableRows.add(result.getRow());
+      }
+      MetaEditor.removeRegionReplicasFromMeta(tableRows, newReplicaCount,
+          oldReplicaCount - newReplicaCount, masterServices.getCatalogTracker());
+    } finally {
+      if (htable != null) {
+        htable.close();
+      }
+    }
+  }
+
   /**
    * Removes from hdfs the families that are not longer present in the new table descriptor.
    */

Modified: hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java?rev=1570008&r1=1570007&r2=1570008&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java
(original)
+++ hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java
Thu Feb 20 00:26:56 2014
@@ -181,44 +181,43 @@ public class TestMasterOperationsForRegi
 //        TEST_UTIL.getMiniHBaseCluster().startRegionServer();
 //      }
 
-      //TODO: HBASE-10361 patch should uncomment the test below
-//      //check on alter table
-//      admin.disableTable(table);
-//      assert(admin.isTableDisabled(table));
-//      //increase the replica
-//      desc.setRegionReplication(numReplica + 1);
-//      admin.modifyTable(table, desc);
-//      admin.enableTable(table);
-//      assert(admin.isTableEnabled(table));
-//      List<HRegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster()
-//          .getAssignmentManager().getRegionStates().getRegionsOfTable(table);
-//      assert(regions.size() == numRegions * (numReplica + 1));
-//
-//      //decrease the replica(earlier, table was modified to have a replica count of numReplica
+ 1)
-//      admin.disableTable(table);
-//      desc.setRegionReplication(numReplica);
-//      admin.modifyTable(table, desc);
-//      admin.enableTable(table);
-//      assert(admin.isTableEnabled(table));
-//      regions = TEST_UTIL.getMiniHBaseCluster().getMaster()
-//          .getAssignmentManager().getRegionStates().getRegionsOfTable(table);
-//      assert(regions.size() == numRegions * numReplica);
-//      //also make sure the meta table has the replica locations removed
-//      hris = MetaReader.getTableRegions(ct, table);
-//      assert(hris.size() == numRegions * numReplica);
-//      //just check that the number of default replica regions in the meta table are the
same
-//      //as the number of regions the table was created with, and the count of the
-//      //replicas is numReplica for each region
-//      Map<HRegionInfo, Integer> defaultReplicas = new HashMap<HRegionInfo, Integer>();
-//      for (HRegionInfo hri : hris) {
-//        Integer i;
-//        HRegionInfo regionReplica0 = hri.getRegionInfoForReplica(0);
-//        defaultReplicas.put(regionReplica0, 
-//            (i = defaultReplicas.get(regionReplica0)) == null ? 1 : i + 1);
-//      }
-//      assert(defaultReplicas.size() == numRegions);
-//      Collection<Integer> counts = new HashSet<Integer>(defaultReplicas.values());
-//      assert(counts.size() == 1 && counts.contains(new Integer(numReplica)));
+      //check on alter table
+      admin.disableTable(table);
+      assert(admin.isTableDisabled(table));
+      //increase the replica
+      desc.setRegionReplication(numReplica + 1);
+      admin.modifyTable(table, desc);
+      admin.enableTable(table);
+      assert(admin.isTableEnabled(table));
+      List<HRegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster()
+          .getAssignmentManager().getRegionStates().getRegionsOfTable(table);
+      assert(regions.size() == numRegions * (numReplica + 1));
+
+      //decrease the replica(earlier, table was modified to have a replica count of numReplica
+ 1)
+      admin.disableTable(table);
+      desc.setRegionReplication(numReplica);
+      admin.modifyTable(table, desc);
+      admin.enableTable(table);
+      assert(admin.isTableEnabled(table));
+      regions = TEST_UTIL.getMiniHBaseCluster().getMaster()
+          .getAssignmentManager().getRegionStates().getRegionsOfTable(table);
+      assert(regions.size() == numRegions * numReplica);
+      //also make sure the meta table has the replica locations removed
+      hris = MetaReader.getTableRegions(ct, table);
+      assert(hris.size() == numRegions * numReplica);
+      //just check that the number of default replica regions in the meta table are the same
+      //as the number of regions the table was created with, and the count of the
+      //replicas is numReplica for each region
+      Map<HRegionInfo, Integer> defaultReplicas = new HashMap<HRegionInfo, Integer>();
+      for (HRegionInfo hri : hris) {
+        Integer i;
+        HRegionInfo regionReplica0 = RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
+        defaultReplicas.put(regionReplica0, 
+            (i = defaultReplicas.get(regionReplica0)) == null ? 1 : i + 1);
+      }
+      assert(defaultReplicas.size() == numRegions);
+      Collection<Integer> counts = new HashSet<Integer>(defaultReplicas.values());
+      assert(counts.size() == 1 && counts.contains(new Integer(numReplica)));
     } finally {
       admin.disableTable(table);
       admin.deleteTable(table);

Modified: hbase/branches/hbase-10070/hbase-shell/src/main/ruby/hbase/admin.rb
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-shell/src/main/ruby/hbase/admin.rb?rev=1570008&r1=1570007&r2=1570008&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-shell/src/main/ruby/hbase/admin.rb (original)
+++ hbase/branches/hbase-10070/hbase-shell/src/main/ruby/hbase/admin.rb Thu Feb 20 00:26:56
2014
@@ -478,6 +478,7 @@ module Hbase
         htd.setMemStoreFlushSize(JLong.valueOf(arg.delete(MEMSTORE_FLUSHSIZE))) if arg[MEMSTORE_FLUSHSIZE]
         htd.setAsyncLogFlush(JBoolean.valueOf(arg.delete(DEFERRED_LOG_FLUSH))) if arg[DEFERRED_LOG_FLUSH]
         htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf(arg.delete(DURABILITY)))
if arg[DURABILITY]
+        htd.setRegionReplication(JInteger.valueOf(arg.delete(REGION_REPLICATION))) if arg[REGION_REPLICATION]
         set_user_metadata(htd, arg.delete(METADATA)) if arg[METADATA]
         set_descriptor_config(htd, arg.delete(CONFIGURATION)) if arg[CONFIGURATION]
 

Modified: hbase/branches/hbase-10070/hbase-shell/src/main/ruby/shell/commands/alter.rb
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-shell/src/main/ruby/shell/commands/alter.rb?rev=1570008&r1=1570007&r2=1570008&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-shell/src/main/ruby/shell/commands/alter.rb (original)
+++ hbase/branches/hbase-10070/hbase-shell/src/main/ruby/shell/commands/alter.rb Thu Feb 20
00:26:56 2014
@@ -74,6 +74,10 @@ You can also remove a table-scope attrib
 
   hbase> alter 't1', METHOD => 'table_att_unset', NAME => 'coprocessor$1'
 
+You can also set REGION_REPLICATION:
+
+  hbase> alter 't1', {REGION_REPLICATION => 2}
+
 There could be more than one alteration in one command:
 
   hbase> alter 't1', { NAME => 'f1', VERSIONS => 3 }, 



Mime
View raw message