accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dlmar...@apache.org
Subject accumulo git commit: Addressed some comments about using the same comparator as the current set of tservers and allowing multiple pools per host
Date Wed, 30 Mar 2016 18:08:03 GMT
Repository: accumulo
Updated Branches:
  refs/heads/ACCUMULO-4173 796cbcf7f -> 47679074f


Addressed some comments about using the same comparator as the current set of tservers and
allowing multiple pools per host


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/47679074
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/47679074
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/47679074

Branch: refs/heads/ACCUMULO-4173
Commit: 47679074f1ef784708d1aab86f292c64dcd99178
Parents: 796cbcf
Author: Dave Marion <dlmarion@apache.org>
Authored: Wed Mar 30 14:06:43 2016 -0400
Committer: Dave Marion <dlmarion@apache.org>
Committed: Wed Mar 30 14:06:43 2016 -0400

----------------------------------------------------------------------
 .../balancer/HostRegexTableLoadBalancer.java    | 78 ++++++++------------
 .../HostRegexTableLoadBalancerTest.java         | 40 ++++++----
 2 files changed, 54 insertions(+), 64 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/47679074/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
index 8e28132..74154a5 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
@@ -91,32 +91,12 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer {
     if ((System.currentTimeMillis() - lastPoolRecheck) > poolRecheckMillis) {
       Map<String,SortedMap<TServerInstance,TabletServerStatus>> newPools = new
HashMap<String,SortedMap<TServerInstance,TabletServerStatus>>();
       for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
-        String tableName = getPoolNameForHost(e.getKey().host());
-        if (!newPools.containsKey(tableName)) {
-          newPools.put(tableName, new TreeMap<TServerInstance,TabletServerStatus>());
-        }
-        newPools.get(tableName).put(e.getKey(), e.getValue());
-      }
-      // Ensure that no host is in more than one pool
-      // TODO: I'm not sure that I need to check for disjoint as the call to getPoolNameForHost
checks for more than one match
-      boolean error = false;
-      for (SortedMap<TServerInstance,TabletServerStatus> s1 : newPools.values()) {
-        for (SortedMap<TServerInstance,TabletServerStatus> s2 : newPools.values())
{
-          if (s1 == s2) {
-            continue;
-          }
-          if (!Collections.disjoint(s1.keySet(), s2.keySet())) {
-            LOG.error("Pools are not disjoint: {}, there is a problem with your regular expressions.
Putting all servers in default pool", newPools);
-            error = true;
+        List<String> poolNames = getPoolNamesForHost(e.getKey().host());
+        for (String pool : poolNames) {
+          if (!newPools.containsKey(pool)) {
+            newPools.put(pool, new TreeMap<TServerInstance,TabletServerStatus>(current.comparator()));
           }
-        }
-      }
-      if (error) {
-        // Put all servers into the default pool
-        newPools.clear();
-        newPools.put(DEFAULT_POOL, new TreeMap<TServerInstance,TabletServerStatus>());
-        for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
-          newPools.get(DEFAULT_POOL).put(e.getKey(), e.getValue());
+          newPools.get(pool).put(e.getKey(), e.getValue());
         }
       }
       pools = newPools;
@@ -126,33 +106,32 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer {
   }
 
   /**
-   * Matches host against the regexes and returns the matching pool name
+   * Matches host against the regexes and returns the matching pool names
    *
    * @param host
    *          tablet server host
-   * @return name of pool. will return default pool if host matches more than one regex
+   * @return pool names, will return default pool if host matches more no regex
    */
-  protected String getPoolNameForHost(String host) {
+  protected List<String> getPoolNamesForHost(String host) {
     String test = host;
-    String table = DEFAULT_POOL;
     if (!isIpBasedRegex) {
       try {
         test = getNameFromIp(host);
       } catch (UnknownHostException e1) {
         LOG.error("Unable to determine host name for IP: " + host + ", setting to default
pool", e1);
-        return table;
+        return Collections.singletonList(DEFAULT_POOL);
       }
     }
+    List<String> pools = new ArrayList<>();
     for (Entry<String,Pattern> e : poolNameToRegexPattern.entrySet()) {
       if (e.getValue().matcher(test).matches()) {
-        if (!table.equals(DEFAULT_POOL)) {
-          LOG.warn("host {} matches more than one regex, assigning to default pool", host);
-          return DEFAULT_POOL;
-        }
-        table = e.getKey();
+        pools.add(e.getKey());
       }
     }
-    return table;
+    if (pools.size() == 0) {
+      pools.add(DEFAULT_POOL);
+    }
+    return pools;
   }
 
   protected String getNameFromIp(String hostIp) throws UnknownHostException {
@@ -240,20 +219,21 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer {
     if ((System.currentTimeMillis() - this.lastOOBCheck) > this.oobCheckMillis) {
       // Check to see if a tablet is assigned outside the bounds of the pool. If so, migrate
it.
       for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
-        String assignedPool = getPoolNameForHost(e.getKey().host());
-        for (String pool : poolNameToRegexPattern.keySet()) {
-          if (assignedPool.equals(pool) || pool.equals(DEFAULT_POOL)) {
-            continue;
-          }
-          String tid = getTableOperations().tableIdMap().get(pool);
-          try {
-            List<TabletStats> outOfBoundsTablets = getOnlineTabletsForTable(e.getKey(),
tid);
-            for (TabletStats ts : outOfBoundsTablets) {
-              LOG.info("Tablet {} is currently outside the bounds of the regex, reassigning",
ts.toString());
-              unassigned.put(new KeyExtent(ts.getExtent()), e.getKey());
+        for (String assignedPool : getPoolNamesForHost(e.getKey().host())) {
+          for (String pool : poolNameToRegexPattern.keySet()) {
+            if (assignedPool.equals(pool) || pool.equals(DEFAULT_POOL)) {
+              continue;
+            }
+            String tid = getTableOperations().tableIdMap().get(pool);
+            try {
+              List<TabletStats> outOfBoundsTablets = getOnlineTabletsForTable(e.getKey(),
tid);
+              for (TabletStats ts : outOfBoundsTablets) {
+                LOG.info("Tablet {} is currently outside the bounds of the regex, reassigning",
ts.toString());
+                unassigned.put(new KeyExtent(ts.getExtent()), e.getKey());
+              }
+            } catch (TException e1) {
+              LOG.error("Error in OOB check getting tablets for table {} from server {}",
tid, e.getKey().host(), e);
             }
-          } catch (TException e1) {
-            LOG.error("Error in OOB check getting tablets for table {} from server {}", tid,
e.getKey().host(), e);
           }
         }
       }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/47679074/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
index e14b511..c33752a 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
@@ -344,7 +344,7 @@ public class HostRegexTableLoadBalancerTest extends HostRegexTableLoadBalancer
{
   }
 
   @Test
-  public void testSplitCurrentByRegexUsingHostnameMatchesTooMany() {
+  public void testSplitCurrentByRegexUsingOverlappingPools() {
     init((ServerConfiguration) new TestServerConfigurationFactory(instance) {
       @Override
       public TableConfiguration getTableConfiguration(String tableId) {
@@ -375,25 +375,35 @@ public class HostRegexTableLoadBalancerTest extends HostRegexTableLoadBalancer
{
     Assert.assertEquals(2, groups.size());
     Assert.assertTrue(groups.containsKey(FOO.getTableName()));
     SortedMap<TServerInstance,TabletServerStatus> fooHosts = groups.get(FOO.getTableName());
-    Assert.assertEquals(5, fooHosts.size());
+    Assert.assertEquals(15, fooHosts.size());
+    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.1:9997", 1)));
+    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.2:9997", 1)));
+    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.3:9997", 1)));
+    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.4:9997", 1)));
+    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.5:9997", 1)));
+    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.6:9997", 1)));
+    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.7:9997", 1)));
+    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.8:9997", 1)));
+    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.9:9997", 1)));
+    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.10:9997", 1)));
     Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.11:9997", 1)));
     Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.12:9997", 1)));
     Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.13:9997", 1)));
     Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.14:9997", 1)));
     Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.15:9997", 1)));
-    Assert.assertTrue(groups.containsKey(DEFAULT_POOL));
-    SortedMap<TServerInstance,TabletServerStatus> defHosts = groups.get(DEFAULT_POOL);
-    Assert.assertEquals(10, defHosts.size());
-    Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.1:9997", 1)));
-    Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.2:9997", 1)));
-    Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.3:9997", 1)));
-    Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.4:9997", 1)));
-    Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.5:9997", 1)));
-    Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.6:9997", 1)));
-    Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.7:9997", 1)));
-    Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.8:9997", 1)));
-    Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.9:9997", 1)));
-    Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.10:9997", 1)));
+    Assert.assertTrue(groups.containsKey(BAR.getTableName()));
+    SortedMap<TServerInstance,TabletServerStatus> barHosts = groups.get(BAR.getTableName());
+    Assert.assertEquals(10, barHosts.size());
+    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.1:9997", 1)));
+    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.2:9997", 1)));
+    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.3:9997", 1)));
+    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.4:9997", 1)));
+    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.5:9997", 1)));
+    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.6:9997", 1)));
+    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.7:9997", 1)));
+    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.8:9997", 1)));
+    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.9:9997", 1)));
+    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.10:9997", 1)));
   }
 
   @Test


Mime
View raw message