hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject svn commit: r893638 - in /hadoop/hbase/branches/0.20_on_hadoop-0.18.3: ./ src/contrib/ src/contrib/ec2/bin/ src/java/org/apache/hadoop/hbase/client/ src/java/org/apache/hadoop/hbase/master/ src/test/org/apache/hadoop/hbase/ src/test/org/apache/hadoop/h...
Date Wed, 23 Dec 2009 22:11:05 GMT
Author: apurtell
Date: Wed Dec 23 22:11:04 2009
New Revision: 893638

URL: http://svn.apache.org/viewvc?rev=893638&view=rev
Log:
fixes for 2064, 2065, and 1982

Added:
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/master/TestMinimumServerCount.java
Modified:
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/CHANGES.txt
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/build-contrib.xml
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/launch-hbase-zookeeper
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnection.java
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/HMaster.java
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/RegionManager.java
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/ServerManager.java
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/CHANGES.txt?rev=893638&r1=893637&r2=893638&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/CHANGES.txt Wed Dec 23 22:11:04 2009
@@ -24,6 +24,10 @@
                hbase.version
    HBASE-2054  memstore size 0 is >= than blocking -2.0g size
    HBASE-2057  Cluster won't stop 
+   HBASE-2064  Cannot disable a table if at the same the Master is moving 
+               its regions around
+   HBASE-2065  Cannot disable a table if any of its region is opening 
+               at the same time
 
   IMPROVEMENTS
    HBASE-1961  HBase EC2 scripts
@@ -52,6 +56,8 @@
                Andrew Purtell)
    HBASE-2028  Add HTable.incrementColumnValue support to shell (Lars George
                via Andrew Purtell)
+   HBASE-1982  [EC2] Handle potentially large and uneven instance startup
+               times
 
 Release 0.20.2 - November 18th, 2009
   INCOMPATIBLE CHANGES

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/build-contrib.xml
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/build-contrib.xml?rev=893638&r1=893637&r2=893638&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/build-contrib.xml (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/build-contrib.xml Wed Dec 23 22:11:04
2009
@@ -21,7 +21,7 @@
 
 <project name="hbasebuildcontrib">
 
-  <property name="version" value="0.20.2-0.18.3"/>
+  <property name="version" value="0.20.0"/>
   <property name="name" value="${ant.project.name}"/>
   <property name="root" value="${basedir}"/>
 

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/launch-hbase-zookeeper
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/launch-hbase-zookeeper?rev=893638&r1=893637&r2=893638&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/launch-hbase-zookeeper
(original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/launch-hbase-zookeeper
Wed Dec 23 22:11:04 2009
@@ -77,8 +77,8 @@
 
 # Start Zookeeper quorum
 
+sleep 5
 echo "Initializing the ZooKeeper quorum ensemble."
-
 for host in $public_names ; do 
   echo "    $host"
   scp $SSH_OPTS "$bin"/hbase-ec2-init-zookeeper-remote.sh "root@${host}:/var/tmp"

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java?rev=893638&r1=893637&r2=893638&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
(original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
Wed Dec 23 22:11:04 2009
@@ -441,6 +441,24 @@
   }
 
   /**
+   * @param tableName name of table to check
+   * @return true if all regions of the table are available
+   * @throws IOException
+   */
+  public boolean isTableAvailable(byte[] tableName) throws IOException {
+    return connection.isTableAvailable(tableName);
+  }
+
+  /**
+   * @param tableName name of table to check
+   * @return true if all regions of the table are available
+   * @throws IOException
+   */
+  public boolean isTableAvailable(String tableName) throws IOException {
+    return connection.isTableAvailable(Bytes.toBytes(tableName));
+  }
+
+  /**
    * Add a column to an existing table.
    * Asynchronous operation.
    * 

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnection.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnection.java?rev=893638&r1=893637&r2=893638&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnection.java
(original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnection.java
Wed Dec 23 22:11:04 2009
@@ -76,7 +76,14 @@
    * @throws IOException
    */
   public boolean isTableDisabled(byte[] tableName) throws IOException;
-  
+
+  /**
+   * @param tableName
+   * @return true if all regions of the table are available, false otherwise
+   * @throws IOException
+   */
+  public boolean isTableAvailable(byte[] tableName) throws IOException;
+
   /**
    * List all the userspace tables.  In other words, scan the META table.
    *

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=893638&r1=893637&r2=893638&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
(original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
Wed Dec 23 22:11:04 2009
@@ -29,6 +29,7 @@
 import java.util.Map;
 import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -463,6 +464,29 @@
       return testTableOnlineState(tableName, false);
     }
 
+    public boolean isTableAvailable(final byte[] tableName) throws IOException {
+      final AtomicBoolean available = new AtomicBoolean(true);
+      MetaScannerVisitor visitor = new MetaScannerVisitor() {
+        @Override
+        public boolean processRow(Result row) throws IOException {
+          byte[] value = row.getValue(CATALOG_FAMILY, REGIONINFO_QUALIFIER);
+          HRegionInfo info = Writables.getHRegionInfoOrNull(value);
+          if (info != null) {
+            if (Bytes.equals(tableName, info.getTableDesc().getName())) {
+              value = row.getValue(CATALOG_FAMILY, SERVER_QUALIFIER);
+              if (value == null) {
+                available.set(false);
+                return false;
+              }
+            }
+          }
+          return true;
+        }        
+      };
+      MetaScanner.metaScan(conf, visitor);
+      return available.get();
+    }
+
     /*
      * If online == true
      *   Returns true if all regions are online

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/HMaster.java?rev=893638&r1=893637&r2=893638&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/HMaster.java
(original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/HMaster.java
Wed Dec 23 22:11:04 2009
@@ -396,6 +396,13 @@
     return regionManager.getOnlineMetaRegions();
   }
 
+  /**
+   * @return the server manager instance
+   */
+  public ServerManager getServerManager() {
+    return this.serverManager;
+  }
+
   /** Main processing loop */
   @Override
   public void run() {
@@ -744,6 +751,9 @@
         if (!regionManager.areAllMetaRegionsOnline()) {
           throw new NotAllMetaRegionsOnlineException();
         }
+        if (!this.serverManager.canAssignUserRegions()) {
+          throw new IOException("not enough servers to create table yet");
+        }
         createTable(newRegion);
         LOG.info("created table " + desc.getNameAsString());
         break;
@@ -1174,7 +1184,11 @@
    */
 
   private static void printUsageAndExit() {
-    System.err.println("Usage: java org.apache.hbase.HMaster start|stop");
+    System.err.println("Usage: Master [opts] start|stop");
+    System.err.println(" start  Start Master. If local mode, start Master and RegionServer
in same JVM");
+    System.err.println(" stop   Start cluster shutdown; Master signals RegionServer shutdown");
+    System.err.println(" where [opts] are:");
+    System.err.println("   --minServers=<servers>    Minimum RegionServers needed to
host user tables.");
     System.exit(0);
   }
 
@@ -1191,6 +1205,12 @@
     // (but hopefully something not as painful as cli options).
     for (String cmd: args) {
 
+      if (cmd.startsWith("--minServers=")) {
+        conf.setInt("hbase.regions.server.count.min",
+          Integer.valueOf(cmd.substring(13)));
+        continue;
+      }
+
       if (cmd.equals("start")) {
         try {
           RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean();

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java?rev=893638&r1=893637&r2=893638&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java
(original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java
Wed Dec 23 22:11:04 2009
@@ -103,8 +103,16 @@
           master.regionManager.metaScannerThread.interrupt();
         }
       }
-      // If updated successfully, remove from pending list.
-      master.regionManager.removeRegion(regionInfo);
+      // If updated successfully, remove from pending list if the state
+      // is consistent. For example, a disable could be called before the
+      // synchronization.
+      if(master.regionManager.
+        isOfflined(regionInfo.getRegionNameAsString())) {
+        LOG.warn("We opened a region while it was asked to be closed.");
+      } else {
+        master.regionManager.removeRegion(regionInfo);
+      }
+
       return true;
     }
   }

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/RegionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/RegionManager.java?rev=893638&r1=893637&r2=893638&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/RegionManager.java
(original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/RegionManager.java
Wed Dec 23 22:11:04 2009
@@ -56,7 +56,6 @@
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.Writables;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
 
 /**
  * Class to manage assigning regions to servers, state of root and meta, etc.
@@ -424,6 +423,12 @@
         // and are on-line
         continue;
       }
+      if (!i.isMetaRegion() && 
+          !master.getServerManager().canAssignUserRegions()) {
+        LOG.debug("user region " + i.getRegionNameAsString() +
+          " is in transition but not enough servers yet");
+        continue;
+      }
       if (s.isUnassigned()) {
         regionsToAssign.add(s);
       }
@@ -952,7 +957,7 @@
         s = new RegionState(info);
         regionsInTransition.put(info.getRegionNameAsString(), s);
       }
-      if (force || (!s.isPendingOpen() && !s.isOpen())) {
+      if (force || (!s.isPendingOpen() || !s.isOpen())) {
         s.setUnassigned();
       }
     }
@@ -1025,7 +1030,7 @@
    * @param regionInfo
    * @param setOffline
    */
-  public void setClosing(final String serverName, final HRegionInfo regionInfo,
+  public void setClosing(String serverName, final HRegionInfo regionInfo,
       final boolean setOffline) {
     synchronized (this.regionsInTransition) {
       RegionState s =
@@ -1033,6 +1038,12 @@
       if (s == null) {
         s = new RegionState(regionInfo);
       }
+      // If region was asked to open before getting here, we could be taking
+      // the wrong server name
+      if(s.isPendingOpen()) {
+        serverName = s.getServerName();
+      }
+
       s.setClosing(serverName, setOffline);
       this.regionsInTransition.put(regionInfo.getRegionNameAsString(), s);
     }
@@ -1635,10 +1646,10 @@
     }
     
     synchronized void setClosed() {
-      if (!pendingClose && !pendingOpen) {
+      if (!pendingClose && !pendingOpen && !closing) {
         throw new IllegalStateException(
             "Cannot set a region to be closed if it was not already marked as" +
-            " pending close or pending open. State: " + toString());
+            " pending close, pending open or closing. State: " + toString());
       }
       this.unassigned = false;
       this.pendingOpen = false;

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/ServerManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/ServerManager.java?rev=893638&r1=893637&r2=893638&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/ServerManager.java
(original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/ServerManager.java
Wed Dec 23 22:11:04 2009
@@ -35,6 +35,7 @@
 import org.apache.commons.logging.LogFactory;
 
 import org.apache.hadoop.hbase.Chore;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HServerInfo;
 import org.apache.hadoop.hbase.HServerLoad;
 import org.apache.hadoop.hbase.HServerAddress;
@@ -48,7 +49,6 @@
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
 import org.apache.zookeeper.WatchedEvent;
 import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.Watcher.Event.EventType;
@@ -139,11 +139,12 @@
    */
   public ServerManager(HMaster master) {
     this.master = master;
-    this.nobalancingCount = master.getConfiguration().
-      getInt("hbase.regions.nobalancing.count", 4);
-    serverMonitorThread = new ServerMonitor(master.metaRescanInterval,
+    HBaseConfiguration c = master.getConfiguration();
+    this.nobalancingCount = c.getInt("hbase.regions.nobalancing.count", 4);
+    this.minimumServerCount = c.getInt("hbase.regions.server.count.min", 0);
+    this.serverMonitorThread = new ServerMonitor(master.metaRescanInterval,
       master.shutdownRequested);
-    serverMonitorThread.start();
+    this.serverMonitorThread.start();
   }
  
   /**
@@ -413,6 +414,8 @@
     return processMsgs(serverInfo, mostLoadedRegions, msgs);
   }
 
+  private int minimumServerCount;
+
   /*
    * Process all the incoming messages from a server that's contacted us.
    * Note that we never need to update the server's load information because
@@ -867,4 +870,16 @@
   public boolean isDead(String serverName) {
     return deadServers.contains(serverName);
   }
+
+  public boolean canAssignUserRegions() {
+    if (minimumServerCount == 0) {
+      return true;
+    }
+    return (numServers() >= minimumServerCount);
+  }
+
+  public void setMinimumServerCount(int minimumServerCount) {
+    this.minimumServerCount = minimumServerCount;
+  }
+
 }

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java?rev=893638&r1=893637&r2=893638&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java
(original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java
Wed Dec 23 22:11:04 2009
@@ -39,7 +39,7 @@
   static final Log LOG = LogFactory.getLog(MiniHBaseCluster.class.getName());
   
   private HBaseConfiguration conf;
-  private LocalHBaseCluster hbaseCluster;
+  public LocalHBaseCluster hbaseCluster;
 
   /**
    * Start a MiniHBaseCluster. 

Added: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/master/TestMinimumServerCount.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/master/TestMinimumServerCount.java?rev=893638&view=auto
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/master/TestMinimumServerCount.java
(added)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/master/TestMinimumServerCount.java
Wed Dec 23 22:11:04 2009
@@ -0,0 +1,68 @@
+package org.apache.hadoop.hbase.master;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.HBaseClusterTestCase;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+
+public class TestMinimumServerCount extends HBaseClusterTestCase {
+
+  static final String TABLE_NAME = "TestTable";
+
+  public TestMinimumServerCount() {
+    // start cluster with one region server only
+    super(1, true);
+  }
+
+  boolean isTableAvailable(String tableName) throws IOException {
+    boolean available = true;
+    HTable meta = new HTable(conf, ".META.");
+    ResultScanner scanner = meta.getScanner(HConstants.CATALOG_FAMILY);
+    Result result;
+    while ((result = scanner.next()) != null) {
+      // set available to false if a region of the table is found with no
+      // assigned server
+      byte[] value = result.getValue(HConstants.CATALOG_FAMILY,
+        HConstants.SERVER_QUALIFIER);
+      if (value == null) {
+        available = false;
+        break;
+      }
+    }
+    return available;
+  }
+
+  public void testMinimumServerCount() throws Exception {
+    HBaseAdmin admin = new HBaseAdmin(conf);
+
+    // create and disable table
+    admin.createTable(createTableDescriptor(TABLE_NAME));
+    admin.disableTable(TABLE_NAME);
+    assertFalse(admin.isTableEnabled(TABLE_NAME));
+
+    // reach in and set minimum server count
+    cluster.hbaseCluster.getMaster().getServerManager()
+      .setMinimumServerCount(2);
+
+    // now try to enable the table
+    try {
+      admin.enableTable(TABLE_NAME);
+    } catch (IOException ex) {
+      // ignore
+    }
+    Thread.sleep(10 * 1000);
+    assertFalse(admin.isTableAvailable(TABLE_NAME));
+    
+    // now start another region server
+    cluster.startRegionServer();
+
+    // sleep a bit for assignment
+    Thread.sleep(10 * 1000);
+    assertTrue(admin.isTableAvailable(TABLE_NAME));
+  }
+
+}



Mime
View raw message