hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r989883 - in /hbase/branches/0.90_master_rewrite/src: main/java/org/apache/hadoop/hbase/master/ main/java/org/apache/hadoop/hbase/regionserver/ test/java/org/apache/hadoop/hbase/ test/java/org/apache/hadoop/hbase/avro/ test/java/org/apache/...
Date Thu, 26 Aug 2010 19:14:46 GMT
Author: stack
Date: Thu Aug 26 19:14:45 2010
New Revision: 989883

URL: http://svn.apache.org/viewvc?rev=989883&view=rev
Log:

Made the balancer test pass (About 3 or 4 small bugs in balancer)

All tests but an avro, a rest and a WAL test are passing now.

Starting merge from TRUNK


Modified:
    hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
    hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
    hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
    hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
    hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
    hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java
    hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
    hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/avro/TestAvroServer.java
    hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java

Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java?rev=989883&r1=989882&r2=989883&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
(original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
Thu Aug 26 19:14:45 2010
@@ -1034,7 +1034,9 @@ public class AssignmentManager extends Z
   }
 
   /**
-   * @return A clone of current assignments
+   * @return A clone of current assignments. Note, this is assignments only.
+   * If a new server has come in and it has no regions, it will not be included
+   * in the returned Map.
    */
   Map<HServerInfo, List<HRegionInfo>> getAssignments() {
     // This is an EXPENSIVE clone.  Cloning though is the safest thing to do.
@@ -1046,7 +1048,11 @@ public class AssignmentManager extends Z
       result = new HashMap<HServerInfo, List<HRegionInfo>>(this.servers.size());
       for (Map.Entry<HServerInfo, List<HRegionInfo>> e: this.servers.entrySet())
{
         List<HRegionInfo> shallowCopy = new ArrayList<HRegionInfo>(e.getValue());
-        result.put(e.getKey(), shallowCopy);
+        HServerInfo clone = new HServerInfo(e.getKey());
+        // Set into server load the number of regions this server is carrying
+        // The load balancer calculation needs it at least and its handy.
+        clone.getLoad().setNumberOfRegions(e.getValue().size());
+        result.put(clone, shallowCopy);
       }
     }
     return result;

Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=989883&r1=989882&r2=989883&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
(original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
Thu Aug 26 19:14:45 2010
@@ -26,6 +26,7 @@ import java.lang.management.RuntimeMXBea
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 import java.net.UnknownHostException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
@@ -537,6 +538,14 @@ implements HMasterInterface, HMasterRegi
       }
       Map<HServerInfo, List<HRegionInfo>> assignments =
         this.assignmentManager.getAssignments();
+      // Returned Map from AM does not include mention of servers w/o assignments.
+      for (Map.Entry<String, HServerInfo> e:
+          this.serverManager.getOnlineServers().entrySet()) {
+        HServerInfo hsi = e.getValue();
+        if (!assignments.containsKey(hsi)) {
+          assignments.put(hsi, new ArrayList<HRegionInfo>());
+        }
+      }
       List<RegionPlan> plans = this.balancer.balanceCluster(assignments);
       if (plans == null || plans.isEmpty()) return;
       for (RegionPlan plan: plans) {

Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java?rev=989883&r1=989882&r2=989883&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
(original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
Thu Aug 26 19:14:45 2010
@@ -141,6 +141,7 @@ public class LoadBalancer {
     // Iterate so we can count regions as we build the map
     for(Map.Entry<HServerInfo, List<HRegionInfo>> server :
       clusterState.entrySet()) {
+      server.getKey().getLoad().setNumberOfRegions(server.getValue().size());
       numRegions += server.getKey().getLoad().getNumberOfRegions();
       serversByLoad.put(server.getKey(), server.getValue());
     }
@@ -181,11 +182,16 @@ public class LoadBalancer {
       serversOverloaded++;
       List<HRegionInfo> regions = server.getValue();
       int numToOffload = Math.min(regionCount - max, regions.size());
-      for(int i=0; i<numToOffload; i++) {
-        regionsToMove.add(new RegionPlan(regions.get(i), serverInfo, null));
+      int numTaken = 0;
+      for (HRegionInfo hri: regions) {
+        // Don't rebalance meta regions.
+        if (hri.isMetaRegion()) continue;
+        regionsToMove.add(new RegionPlan(hri, serverInfo, null));
+        numTaken++;
+        if (numTaken >= numToOffload) break;
       }
       serverBalanceInfo.put(serverInfo,
-          new BalanceInfo(numToOffload, (-1)*numToOffload));
+          new BalanceInfo(numToOffload, (-1)*numTaken));
     }
 
     // Walk down least loaded, filling each to the min
@@ -235,6 +241,7 @@ public class LoadBalancer {
         int idx =
           balanceInfo == null ? 0 : balanceInfo.getNextRegionForUnload();
         HRegionInfo region = server.getValue().get(idx);
+        if (region.isMetaRegion()) continue; // Don't move meta regions.
         regionsToMove.add(new RegionPlan(region, server.getKey(), null));
         if(--neededRegions == 0) {
           // No more regions needed, done shedding
@@ -559,11 +566,11 @@ public class LoadBalancer {
     }
 
     /**
-     * Get the region name for the region this plan is for.
-     * @return region name
+     * Get the encoded region name for the region this plan is for.
+     * @return Encoded region name
      */
     public String getRegionName() {
-      return this.hri.getRegionNameAsString();
+      return this.hri.getEncodedName();
     }
  
     public HRegionInfo getRegionInfo() {

Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java?rev=989883&r1=989882&r2=989883&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
(original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
Thu Aug 26 19:14:45 2010
@@ -383,9 +383,10 @@ public class ServerManager {
   }
 
   /**
-   * @return Read-only map of servers to serverinfo.
+   * @return Read-only map of servers to serverinfo
    */
   public Map<String, HServerInfo> getOnlineServers() {
+    // Presumption is that iterating the returned Map is OK.
     synchronized (this.onlineServers) {
       return Collections.unmodifiableMap(this.onlineServers);
     }

Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=989883&r1=989882&r2=989883&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
(original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
Thu Aug 26 19:14:45 2010
@@ -288,7 +288,7 @@ public class StoreFile {
    */
   public long getMaxSequenceId() {
     if (this.sequenceid == -1) {
-      throw new IllegalAccessError("Has not been initialized");
+      throw new IllegalAccessError(this + " has not been initialized");
     }
     return this.sequenceid;
   }

Modified: hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java?rev=989883&r1=989882&r2=989883&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java
(original)
+++ hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java
Thu Aug 26 19:14:45 2010
@@ -20,16 +20,19 @@
 
 package org.apache.hadoop.hbase;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import static org.junit.Assert.assertEquals;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.*;
-import static org.junit.Assert.assertEquals;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
 
 public class TestFullLogReconstruction {
 

Modified: hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java?rev=989883&r1=989882&r2=989883&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
(original)
+++ hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
Thu Aug 26 19:14:45 2010
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.regionser
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.JVMClusterUtil;
+import org.apache.hadoop.hbase.util.Threads;
 
 /**
  * Test whether region rebalancing works. (HBASE-71)
@@ -104,17 +105,23 @@ public class TestRegionRebalancing exten
     LOG.debug("Adding 2nd region server.");
     // add a region server - total of 2
     cluster.startRegionServer();
+    while (cluster.getMaster().getServerManager().getOnlineServers().size() < 2) {
+      Threads.sleep(100);
+    }
+    cluster.getMaster().balance();
     assertRegionsAreBalanced();
 
     // add a region server - total of 3
     LOG.debug("Adding 3rd region server.");
     cluster.startRegionServer();
+    cluster.getMaster().balance();
     assertRegionsAreBalanced();
 
     // kill a region server - total of 2
     LOG.debug("Killing the 3rd region server.");
     cluster.stopRegionServer(2, false);
     cluster.waitOnRegionServer(2);
+    cluster.getMaster().balance();
     assertRegionsAreBalanced();
 
     // start two more region servers - total of 4
@@ -122,12 +129,14 @@ public class TestRegionRebalancing exten
     cluster.startRegionServer();
     LOG.debug("Adding 4th region server");
     cluster.startRegionServer();
+    cluster.getMaster().balance();
     assertRegionsAreBalanced();
 
     for (int i = 0; i < 6; i++){
       LOG.debug("Adding " + (i + 5) + "th region server");
       cluster.startRegionServer();
     }
+    cluster.getMaster().balance();
     assertRegionsAreBalanced();
   }
 
@@ -166,7 +175,7 @@ public class TestRegionRebalancing exten
 
       for (HRegionServer server : servers) {
         int serverLoad = server.getOnlineRegions().size();
-        LOG.debug(server.hashCode() + " Avg: " + avg + " actual: " + serverLoad);
+        LOG.debug(server.getServerName() + " Avg: " + avg + " actual: " + serverLoad);
         if (!(avg > 2.0 && serverLoad <= avgLoadPlusSlop
             && serverLoad >= avgLoadMinusSlop)) {
           LOG.debug(server.hashCode() + " Isn't balanced!!! Avg: " + avg +

Modified: hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/avro/TestAvroServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/avro/TestAvroServer.java?rev=989883&r1=989882&r2=989883&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/avro/TestAvroServer.java
(original)
+++ hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/avro/TestAvroServer.java
Thu Aug 26 19:14:45 2010
@@ -22,30 +22,23 @@ import static org.junit.Assert.assertFal
 import static org.junit.Assert.assertTrue;
 
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.hbase.HBaseClusterTestCase;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
 
 import org.apache.avro.Schema;
 import org.apache.avro.generic.GenericArray;
 import org.apache.avro.generic.GenericData;
-
+import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.avro.generated.AColumn;
 import org.apache.hadoop.hbase.avro.generated.AColumnValue;
 import org.apache.hadoop.hbase.avro.generated.AFamilyDescriptor;
 import org.apache.hadoop.hbase.avro.generated.AGet;
 import org.apache.hadoop.hbase.avro.generated.APut;
-import org.apache.hadoop.hbase.avro.generated.AResult;
 import org.apache.hadoop.hbase.avro.generated.ATableDescriptor;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
 
 /**
  * Unit testing for AvroServer.HBaseImpl, a part of the
@@ -129,7 +122,7 @@ public class TestAvroServer {
 
     tableA.maxFileSize = 123456L;
     impl.modifyTable(tableAname, tableA);
-    assertEquals((long) impl.describeTable(tableAname).maxFileSize, 123456L);
+    assertEquals(123456L, (long) impl.describeTable(tableAname).maxFileSize);
 
     impl.enableTable(tableAname);
     assertTrue(impl.isTableEnabled(tableAname));

Modified: hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java?rev=989883&r1=989882&r2=989883&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
(original)
+++ hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
Thu Aug 26 19:14:45 2010
@@ -77,7 +77,7 @@ public class TestRemoteTable extends HBa
       htd.addFamily(new HColumnDescriptor(COLUMN_2));
       htd.addFamily(new HColumnDescriptor(COLUMN_3));
       admin.createTable(htd);
-      HTable table = new HTable(TABLE);
+      HTable table = new HTable(conf, TABLE);
       Put put = new Put(ROW_1);
       put.add(COLUMN_1, QUALIFIER_1, TS_2, VALUE_1);
       table.put(put);



Mime
View raw message