accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ibe...@apache.org
Subject accumulo git commit: ACCUMULO-4654 Continue balancing even with pending migrations
Date Wed, 19 Jul 2017 21:20:48 GMT
Repository: accumulo
Updated Branches:
  refs/heads/1.8 cdafd0203 -> 6c20e50c2


ACCUMULO-4654 Continue balancing even with pending migrations

Added the ability to continue balancing with the HostRegexTableLoadBalancer even if there
are pending migrations.
Also added detection for tables that have been continuously migrating for over an hour.

Closes #272


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6c20e50c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6c20e50c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6c20e50c

Branch: refs/heads/1.8
Commit: 6c20e50c223a44deaccf4e0690ccec9b5749b21c
Parents: cdafd02
Author: Ivan Bella <ivan@bella.name>
Authored: Mon Jun 19 13:57:44 2017 -0400
Committer: Ivan Bella <ivan@bella.name>
Committed: Wed Jul 19 17:20:24 2017 -0400

----------------------------------------------------------------------
 .../balancer/HostRegexTableLoadBalancer.java    | 117 +++++++++++++++++--
 .../BaseHostRegexTableLoadBalancerTest.java     |  65 ++++++++++-
 ...gexTableLoadBalancerReconfigurationTest.java |   2 +-
 .../HostRegexTableLoadBalancerTest.java         |  59 +++++++++-
 4 files changed, 224 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/6c20e50c/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
index 5207230..dbf03d0 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.server.master.balancer;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -31,11 +32,15 @@ import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.regex.Pattern;
 
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
 import org.apache.accumulo.core.client.admin.TableOperations;
 import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.conf.ConfigurationObserver;
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.master.thrift.TableInfo;
 import org.apache.accumulo.core.master.thrift.TabletServerStatus;
 import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
 import org.apache.accumulo.server.conf.ServerConfiguration;
@@ -65,7 +70,9 @@ import org.slf4j.LoggerFactory;
  * <b>table.custom.balancer.host.regex.is.ip=true</b><br>
  * It's possible that this balancer may create a lot of migrations. To limit the number of
migrations that are created during a balance call, set the following
  * property (default 250):<br>
- * <b>table.custom.balancer.host.regex.concurrent.migrations</b>
+ * <b>table.custom.balancer.host.regex.concurrent.migrations</b> This balancer
can continue balancing even if there are outstanding migrations. To limit the
+ * number of outstanding migrations in which this balancer will continue balancing, set the
following property (default 0):<br>
+ * <b>table.custom.balancer.host.regex.max.outstanding.migrations</b>
  *
  */
 public class HostRegexTableLoadBalancer extends TableLoadBalancer implements ConfigurationObserver
{
@@ -81,15 +88,24 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer implements
Con
       + "balancer.host.regex.concurrent.migrations";
   private static final int HOST_BALANCER_REGEX_MAX_MIGRATIONS_DEFAULT = 250;
   protected static final String DEFAULT_POOL = "HostTableLoadBalancer.ALL";
+  private static final int DEFAULT_OUTSTANDING_MIGRATIONS = 0;
+  public static final String HOST_BALANCER_OUTSTANDING_MIGRATIONS_KEY = Property.TABLE_ARBITRARY_PROP_PREFIX.getKey()
+      + "balancer.host.regex.max.outstanding.migrations";
 
   protected long oobCheckMillis = AccumuloConfiguration.getTimeInMillis(HOST_BALANCER_OOB_DEFAULT);
 
+  private static final long ONE_HOUR = 60 * 60 * 1000;
+  private static final Set<KeyExtent> EMPTY_MIGRATIONS = Collections.EMPTY_SET;
+
   private Map<String,String> tableIdToTableName = null;
   private Map<String,Pattern> poolNameToRegexPattern = null;
   private volatile long lastOOBCheck = System.currentTimeMillis();
-  private boolean isIpBasedRegex = false;
+  private volatile boolean isIpBasedRegex = false;
   private Map<String,SortedMap<TServerInstance,TabletServerStatus>> pools = new
HashMap<>();
-  private int maxTServerMigrations = HOST_BALANCER_REGEX_MAX_MIGRATIONS_DEFAULT;
+  private volatile int maxTServerMigrations = HOST_BALANCER_REGEX_MAX_MIGRATIONS_DEFAULT;
+  private volatile int maxOutstandingMigrations = DEFAULT_OUTSTANDING_MIGRATIONS;
+  private final Map<KeyExtent,TabletMigration> migrationsFromLastPass = new HashMap<KeyExtent,TabletMigration>();
+  private final Map<String,Long> tableToTimeSinceNoMigrations = new HashMap<String,Long>();
 
   /**
    * Group the set of current tservers by pool name. Tservers that don't match a regex are
put into a default pool. This could be expensive in the terms of the
@@ -191,7 +207,7 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer implements
Con
         for (Entry<String,String> customProp : customProps.entrySet()) {
           if (customProp.getKey().startsWith(HOST_BALANCER_PREFIX)) {
             if (customProp.getKey().equals(HOST_BALANCER_OOB_CHECK_KEY) || customProp.getKey().equals(HOST_BALANCER_REGEX_USING_IPS_KEY)
-                || customProp.getKey().equals(HOST_BALANCER_REGEX_MAX_MIGRATIONS_KEY)) {
+                || customProp.getKey().equals(HOST_BALANCER_REGEX_MAX_MIGRATIONS_KEY) ||
customProp.getKey().equals(HOST_BALANCER_OUTSTANDING_MIGRATIONS_KEY)) {
               continue;
             }
             String tableName = customProp.getKey().substring(HOST_BALANCER_PREFIX.length());
@@ -213,6 +229,10 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer implements
Con
     if (null != migrations) {
       maxTServerMigrations = Integer.parseInt(migrations);
     }
+    String outstanding = conf.getConfiguration().get(HOST_BALANCER_OUTSTANDING_MIGRATIONS_KEY);
+    if (null != outstanding) {
+      this.maxOutstandingMigrations = Integer.parseInt(outstanding);
+    }
     LOG.info("{}", this);
   }
 
@@ -234,6 +254,14 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer implements
Con
     return poolNameToRegexPattern;
   }
 
+  public int getMaxMigrations() {
+    return maxTServerMigrations;
+  }
+
+  public int getMaxOutstandingMigrations() {
+    return maxOutstandingMigrations;
+  }
+
   public long getOobCheckMillis() {
     return oobCheckMillis;
   }
@@ -301,8 +329,10 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer implements
Con
       return minBalanceTime;
 
     Map<String,String> tableIdMap = t.tableIdMap();
+    long now = System.currentTimeMillis();
+
     Map<String,SortedMap<TServerInstance,TabletServerStatus>> currentGrouped
= splitCurrentByRegex(current);
-    if ((System.currentTimeMillis() - this.lastOOBCheck) > this.oobCheckMillis) {
+    if ((now - this.lastOOBCheck) > this.oobCheckMillis) {
       try {
         // Check to see if a tablet is assigned outside the bounds of the pool. If so, migrate
it.
         for (String table : t.list()) {
@@ -343,7 +373,7 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer implements
Con
                   TServerInstance nextTS = iter.next();
                   LOG.info("Tablet {} is currently outside the bounds of the regex, migrating
from {} to {}", ke, e.getKey(), nextTS);
                   migrationsOut.add(new TabletMigration(ke, e.getKey(), nextTS));
-                  if (migrationsOut.size() > this.maxTServerMigrations) {
+                  if (migrationsOut.size() >= this.maxTServerMigrations) {
                     break;
                   }
                 } else {
@@ -356,6 +386,7 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer implements
Con
           }
         }
       } finally {
+        // this could have taken a while...get a new time
         this.lastOOBCheck = System.currentTimeMillis();
       }
     }
@@ -367,8 +398,34 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer implements
Con
     }
 
     if (migrations != null && migrations.size() > 0) {
-      LOG.warn("Not balancing tables due to {} outstanding migrations", migrations.size());
-      return minBalanceTime;
+      if (migrations.size() >= maxOutstandingMigrations) {
+        LOG.warn("Not balancing tables due to {} outstanding migrations", migrations.size());
+        if (LOG.isTraceEnabled()) {
+          LOG.trace("Sample up to 10 outstanding migrations: {}", Iterables.limit(migrations,
10));
+        }
+        return minBalanceTime;
+      }
+
+      LOG.debug("Current outstanding migrations of {} being applied", migrations.size());
+      if (LOG.isTraceEnabled()) {
+        LOG.trace("Sample up to 10 outstanding migrations: {}", Iterables.limit(migrations,
10));
+      }
+      migrationsFromLastPass.keySet().retainAll(migrations);
+      SortedMap<TServerInstance,TabletServerStatus> currentCopy = new TreeMap(current);
+      Multimap<TServerInstance,String> serverTableIdCopied = HashMultimap.create();
+      for (TabletMigration migration : migrationsFromLastPass.values()) {
+        TableInfo fromInfo = getTableInfo(currentCopy, serverTableIdCopied, migration.tablet.getTableId().toString(),
migration.oldServer);
+        if (fromInfo != null) {
+          fromInfo.setOnlineTablets(fromInfo.getOnlineTablets() - 1);
+        }
+        TableInfo toInfo = getTableInfo(currentCopy, serverTableIdCopied, migration.tablet.getTableId().toString(),
migration.newServer);
+        if (toInfo != null) {
+          toInfo.setOnlineTablets(toInfo.getOnlineTablets() + 1);
+        }
+      }
+      migrations = EMPTY_MIGRATIONS;
+    } else {
+      migrationsFromLastPass.clear();
     }
 
     for (String s : tableIdMap.values()) {
@@ -382,15 +439,57 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer implements
Con
       ArrayList<TabletMigration> newMigrations = new ArrayList<>();
       getBalancerForTable(s).balance(currentView, migrations, newMigrations);
 
+      if (newMigrations.isEmpty()) {
+        tableToTimeSinceNoMigrations.remove(s);
+      } else if (tableToTimeSinceNoMigrations.containsKey(s)) {
+        if ((now - tableToTimeSinceNoMigrations.get(s)) > ONE_HOUR) {
+          LOG.warn("We have been consistently producing migrations for {}: {}", tableName,
Iterables.limit(newMigrations, 10));
+        }
+      } else {
+        tableToTimeSinceNoMigrations.put(s, now);
+      }
+
       migrationsOut.addAll(newMigrations);
-      if (migrationsOut.size() > this.maxTServerMigrations) {
+      if (migrationsOut.size() >= this.maxTServerMigrations) {
         break;
       }
     }
+
+    for (TabletMigration migration : migrationsOut) {
+      migrationsFromLastPass.put(migration.tablet, migration);
+    }
+
     LOG.info("Migrating tablets for balance: {}", migrationsOut);
     return minBalanceTime;
   }
 
+  /**
+   * Get a mutable table info for the specified table and server
+   */
+  private TableInfo getTableInfo(SortedMap<TServerInstance,TabletServerStatus> currentCopy,
Multimap<TServerInstance,String> serverTableIdCopied,
+      String tableId, TServerInstance server) {
+    TableInfo newInfo = null;
+    if (currentCopy.containsKey(server)) {
+      Map<String,TableInfo> newTableMap = currentCopy.get(server).getTableMap();
+      if (newTableMap != null) {
+        newInfo = newTableMap.get(tableId);
+        if (newInfo != null) {
+          Collection<String> tableIdCopied = serverTableIdCopied.get(server);
+          if (tableIdCopied.isEmpty()) {
+            newTableMap = new HashMap<String,TableInfo>(newTableMap);
+            currentCopy.get(server).setTableMap(newTableMap);
+          }
+          if (!tableIdCopied.contains(tableId)) {
+            newInfo = new TableInfo(newInfo);
+            newTableMap.put(tableId, newInfo);
+            tableIdCopied.add(tableId);
+          }
+        }
+      }
+    }
+    return newInfo;
+  }
+
   @Override
   public void propertyChanged(String key) {
     parseConfiguration(this.configuration);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/6c20e50c/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java
b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java
index a1730cb..e44491d 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java
@@ -35,16 +35,20 @@ import org.apache.accumulo.core.client.Instance;
 import org.apache.accumulo.core.client.admin.TableOperations;
 import org.apache.accumulo.core.client.impl.ClientContext;
 import org.apache.accumulo.core.client.impl.TableOperationsImpl;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
 import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.conf.ConfigurationCopy;
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.master.thrift.TableInfo;
 import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
 import org.apache.accumulo.server.conf.ServerConfigurationFactory;
 import org.apache.accumulo.server.conf.TableConfiguration;
 import org.apache.accumulo.server.master.state.TServerInstance;
 import org.apache.hadoop.io.Text;
+import org.apache.thrift.TException;
 import org.easymock.EasyMock;
 
 import com.google.common.base.Predicate;
@@ -138,9 +142,12 @@ public abstract class BaseHostRegexTableLoadBalancerTest extends HostRegexTableL
 
   protected static final HashMap<String,String> DEFAULT_TABLE_PROPERTIES = new HashMap<>();
   {
-    DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_OOB_CHECK_KEY,
"2s");
+    DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_OOB_CHECK_KEY,
"7s");
+    DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_REGEX_MAX_MIGRATIONS_KEY,
"4");
+    DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_OUTSTANDING_MIGRATIONS_KEY,
"10");
     DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + FOO.getTableName(),
"r01.*");
     DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(),
"r02.*");
+    DEFAULT_TABLE_PROPERTIES.put(Property.TABLE_LOAD_BALANCER.getKey(), TestDefaultBalancer.class.getName());
   }
 
   protected static class TestServerConfigurationFactory extends ServerConfigurationFactory
{
@@ -155,7 +162,7 @@ public abstract class BaseHostRegexTableLoadBalancerTest extends HostRegexTableL
     }
 
     @Override
-    public TableConfiguration getTableConfiguration(String tableId) {
+    public TableConfiguration getTableConfiguration(final String tableId) {
       return new TableConfiguration(getInstance(), tableId, null) {
         @Override
         public String get(Property property) {
@@ -174,6 +181,24 @@ public abstract class BaseHostRegexTableLoadBalancerTest extends HostRegexTableL
     }
   }
 
+  protected class TestDefaultBalancer extends DefaultLoadBalancer {
+    @Override
+    public List<TabletStats> getOnlineTabletsForTable(TServerInstance tserver, String
tableId) throws ThriftSecurityException, TException {
+      String tableName = idToTableName(tableId);
+      TServerInstance initialLocation = initialTableLocation.get(tableName);
+      if (tserver.equals(initialLocation)) {
+        List<TabletStats> list = new ArrayList<TabletStats>(5);
+        for (KeyExtent extent : tableExtents.get(tableName)) {
+          TabletStats stats = new TabletStats();
+          stats.setExtent(extent.toThrift());
+          list.add(stats);
+        }
+        return list;
+      }
+      return null;
+    }
+  }
+
   protected static final Table FOO = new Table("foo", "1");
   protected static final Table BAR = new Table("bar", "2");
   protected static final Table BAZ = new Table("baz", "3");
@@ -183,6 +208,7 @@ public abstract class BaseHostRegexTableLoadBalancerTest extends HostRegexTableL
   protected final Map<String,String> servers = new HashMap<>(15);
   protected final SortedMap<TServerInstance,TabletServerStatus> allTabletServers =
new TreeMap<>();
   protected final Map<String,List<KeyExtent>> tableExtents = new HashMap<>(3);
+  protected final Map<String,TServerInstance> initialTableLocation = new HashMap<>(3);
 
   {
     servers.put("192.168.0.1", "r01s01");
@@ -217,6 +243,10 @@ public abstract class BaseHostRegexTableLoadBalancerTest extends HostRegexTableL
     allTabletServers.put(new TServerInstance("192.168.0.14:9997", 1), new TabletServerStatus());
     allTabletServers.put(new TServerInstance("192.168.0.15:9997", 1), new TabletServerStatus());
 
+    initialTableLocation.put(FOO.getTableName(), new TServerInstance("192.168.0.1:9997",
1));
+    initialTableLocation.put(BAR.getTableName(), new TServerInstance("192.168.0.6:9997",
1));
+    initialTableLocation.put(BAZ.getTableName(), new TServerInstance("192.168.0.11:9997",
1));
+
     tableExtents.put(FOO.getTableName(), new ArrayList<KeyExtent>());
     tableExtents.get(FOO.getTableName()).add(new KeyExtent(FOO.getId(), new Text("1"), new
Text("0")));
     tableExtents.get(FOO.getTableName()).add(new KeyExtent(FOO.getId(), new Text("2"), new
Text("1")));
@@ -235,6 +265,7 @@ public abstract class BaseHostRegexTableLoadBalancerTest extends HostRegexTableL
     tableExtents.get(BAZ.getTableName()).add(new KeyExtent(BAZ.getId(), new Text("23"), new
Text("22")));
     tableExtents.get(BAZ.getTableName()).add(new KeyExtent(BAZ.getId(), new Text("24"), new
Text("23")));
     tableExtents.get(BAZ.getTableName()).add(new KeyExtent(BAZ.getId(), new Text("25"), new
Text("24")));
+
   }
 
   protected boolean tabletInBounds(KeyExtent ke, TServerInstance tsi) {
@@ -255,6 +286,18 @@ public abstract class BaseHostRegexTableLoadBalancerTest extends HostRegexTableL
     }
   }
 
+  protected String idToTableName(String id) {
+    if (id.equals(FOO.getId())) {
+      return FOO.getTableName();
+    } else if (id.equals(BAR.getId())) {
+      return BAR.getTableName();
+    } else if (id.equals(BAZ.getId())) {
+      return BAZ.getTableName();
+    } else {
+      return null;
+    }
+  }
+
   @Override
   protected TableOperations getTableOperations() {
     return new TableOperationsImpl(EasyMock.createMock(ClientContext.class)) {
@@ -280,7 +323,7 @@ public abstract class BaseHostRegexTableLoadBalancerTest extends HostRegexTableL
 
   @Override
   protected TabletBalancer getBalancerForTable(String table) {
-    return new DefaultLoadBalancer();
+    return new TestDefaultBalancer();
   }
 
   @Override
@@ -296,7 +339,21 @@ public abstract class BaseHostRegexTableLoadBalancerTest extends HostRegexTableL
     String base = "192.168.0.";
     TreeMap<TServerInstance,TabletServerStatus> current = new TreeMap<>();
     for (int i = 1; i <= numTservers; i++) {
-      current.put(new TServerInstance(base + i + ":9997", 1), new TabletServerStatus());
+      TabletServerStatus status = new TabletServerStatus();
+      Map<String,TableInfo> tableMap = new HashMap<String,TableInfo>();
+      tableMap.put(FOO.getId(), new TableInfo());
+      tableMap.put(BAR.getId(), new TableInfo());
+      tableMap.put(BAZ.getId(), new TableInfo());
+      status.setTableMap(tableMap);
+      current.put(new TServerInstance(base + i + ":9997", 1), status);
+    }
+    // now put all of the tablets on one server
+    for (Map.Entry<String,TServerInstance> entry : initialTableLocation.entrySet())
{
+      TabletServerStatus status = current.get(entry.getValue());
+      if (status != null) {
+        String tableId = getTableOperations().tableIdMap().get(entry.getKey());
+        status.getTableMap().get(tableId).setOnlineTablets(5);
+      }
     }
     return current;
   }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/6c20e50c/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerReconfigurationTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerReconfigurationTest.java
b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerReconfigurationTest.java
index 14f6c3e..1be948f 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerReconfigurationTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerReconfigurationTest.java
@@ -78,7 +78,7 @@ public class HostRegexTableLoadBalancerReconfigurationTest extends BaseHostRegex
     DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(),
"r01.*");
     this.propertiesChanged();
     // Wait to trigger the out of bounds check and the repool check
-    UtilWaitThread.sleep(3000);
+    UtilWaitThread.sleep(10000);
     this.balance(Collections.unmodifiableSortedMap(allTabletServers), migrations, migrationsOut);
     Assert.assertEquals(5, migrationsOut.size());
     for (TabletMigration migration : migrationsOut) {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/6c20e50c/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
index 24054f5..120aab9 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
@@ -36,6 +36,7 @@ import org.apache.accumulo.core.data.thrift.TKeyExtent;
 import org.apache.accumulo.core.master.thrift.TabletServerStatus;
 import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
 import org.apache.accumulo.fate.util.UtilWaitThread;
+import org.apache.accumulo.server.conf.ServerConfiguration;
 import org.apache.accumulo.server.conf.TableConfiguration;
 import org.apache.accumulo.server.master.state.TServerInstance;
 import org.apache.accumulo.server.master.state.TabletMigration;
@@ -50,10 +51,12 @@ public class HostRegexTableLoadBalancerTest extends BaseHostRegexTableLoadBalanc
   @Test
   public void testInit() {
     init(factory);
-    Assert.assertEquals("OOB check interval value is incorrect", 2000, this.getOobCheckMillis());
+    Assert.assertEquals("OOB check interval value is incorrect", 7000, this.getOobCheckMillis());
     @SuppressWarnings("deprecation")
     long poolRecheckMillis = this.getPoolRecheckMillis();
     Assert.assertEquals("Pool check interval value is incorrect", 0, poolRecheckMillis);
+    Assert.assertEquals("Max migrations is incorrect", 4, this.getMaxMigrations());
+    Assert.assertEquals("Max outstanding migrations is incorrect", 10, this.getMaxOutstandingMigrations());
     Assert.assertFalse(isIpBasedRegex());
     Map<String,Pattern> patterns = this.getPoolNameToRegexPattern();
     Assert.assertEquals(2, patterns.size());
@@ -73,12 +76,58 @@ public class HostRegexTableLoadBalancerTest extends BaseHostRegexTableLoadBalanc
   }
 
   @Test
-  public void testBalanceWithMigrations() {
-    List<TabletMigration> migrations = new ArrayList<>();
+  public void testBalance() {
+    init((ServerConfiguration) factory);
+    Set<KeyExtent> migrations = new HashSet<KeyExtent>();
+    List<TabletMigration> migrationsOut = new ArrayList<TabletMigration>();
+    long wait = this.balance(Collections.unmodifiableSortedMap(createCurrent(15)), migrations,
migrationsOut);
+    Assert.assertEquals(20000, wait);
+    // should balance four tablets in one of the tables before reaching max
+    Assert.assertEquals(4, migrationsOut.size());
+
+    // now balance again passing in the new migrations
+    for (TabletMigration m : migrationsOut) {
+      migrations.add(m.tablet);
+    }
+    migrationsOut.clear();
+    wait = this.balance(Collections.unmodifiableSortedMap(createCurrent(15)), migrations,
migrationsOut);
+    Assert.assertEquals(20000, wait);
+    // should balance four tablets in one of the other tables before reaching max
+    Assert.assertEquals(4, migrationsOut.size());
+
+    // now balance again passing in the new migrations
+    for (TabletMigration m : migrationsOut) {
+      migrations.add(m.tablet);
+    }
+    migrationsOut.clear();
+    wait = this.balance(Collections.unmodifiableSortedMap(createCurrent(15)), migrations,
migrationsOut);
+    Assert.assertEquals(20000, wait);
+    // should balance four tablets in one of the other tables before reaching max
+    Assert.assertEquals(4, migrationsOut.size());
+
+    // now balance again passing in the new migrations
+    for (TabletMigration m : migrationsOut) {
+      migrations.add(m.tablet);
+    }
+    migrationsOut.clear();
+    wait = this.balance(Collections.unmodifiableSortedMap(createCurrent(15)), migrations,
migrationsOut);
+    Assert.assertEquals(20000, wait);
+    // no more balancing to do
+    Assert.assertEquals(0, migrationsOut.size());
+  }
+
+  @Test
+  public void testBalanceWithTooManyOutstandingMigrations() {
+    List<TabletMigration> migrationsOut = new ArrayList<>();
     init(factory);
-    long wait = this.balance(Collections.unmodifiableSortedMap(createCurrent(2)), Collections.singleton(new
KeyExtent()), migrations);
+    // lets say we already have migrations ongoing for the FOO and BAR table extends (should
be 5 of each of them) for a total of 10
+    Set<KeyExtent> migrations = new HashSet<KeyExtent>();
+    migrations.addAll(tableExtents.get(FOO.getTableName()));
+    migrations.addAll(tableExtents.get(BAR.getTableName()));
+    long wait = this.balance(Collections.unmodifiableSortedMap(createCurrent(15)), migrations,
migrationsOut);
     Assert.assertEquals(20000, wait);
-    Assert.assertEquals(0, migrations.size());
+    // no migrations should have occurred as 10 is the maxOutstandingMigrations
+    Assert.assertEquals(0, migrationsOut.size());
   }
 
   @Test


Mime
View raw message